From 939f929afe9db02e9a342fd8fddf0e43c359ebfc Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Mon, 6 May 2024 16:05:59 +0800 Subject: [PATCH 001/402] [VL] Add more metrics for generate (#5608) --- .../execution/GenerateExecTransformer.scala | 12 +++++++++++- .../gluten/execution/VeloxMetricsSuite.scala | 15 +++++++++++++++ .../gluten/metrics/GenerateMetricsUpdater.scala | 6 ++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala index 08d2937b5ab7..830fe396b99a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala @@ -51,7 +51,17 @@ case class GenerateExecTransformer( @transient override lazy val metrics = - Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) + Map( + "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), + "numOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), + "numOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of generate"), + "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), + "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), + "numMemoryAllocations" -> SQLMetrics.createMetric( + sparkContext, + "number of memory allocations") + ) override def metricsUpdater(): MetricsUpdater = new GenerateMetricsUpdater(metrics) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala index ac9911bbabc4..ce8450fea423 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala @@ -128,6 +128,21 @@ class VeloxMetricsSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } } + test("Generate metrics") { + runQueryAndCompare("SELECT explode(array(c1, c2, 1)) FROM metrics_t1") { + df => + val generate = find(df.queryExecution.executedPlan) { + case _: GenerateExecTransformer => true + case _ => false + } + assert(generate.isDefined) + val metrics = generate.get.metrics + assert(metrics("numOutputRows").value == 300) + assert(metrics("numOutputVectors").value > 0) + assert(metrics("numOutputBytes").value > 0) + } + } + test("Write metrics") { if (SparkShimLoader.getSparkVersion.startsWith("3.4")) { withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { diff --git a/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala b/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala index 670fd1c4ddfe..0a3dccd6449d 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala @@ -23,6 +23,12 @@ class GenerateMetricsUpdater(val metrics: Map[String, SQLMetric]) extends Metric if (operatorMetrics != null) { val nativeMetrics = operatorMetrics.asInstanceOf[OperatorMetrics] metrics("numOutputRows") += nativeMetrics.outputRows + metrics("numOutputVectors") += nativeMetrics.outputVectors + metrics("numOutputBytes") += nativeMetrics.outputBytes + metrics("cpuCount") += nativeMetrics.cpuCount + metrics("wallNanos") += nativeMetrics.wallNanos + metrics("peakMemoryBytes") += nativeMetrics.peakMemoryBytes + metrics("numMemoryAllocations") += nativeMetrics.numMemoryAllocations } } } From 19248d176fbd24878873ef4e4c9ae572e858c969 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Mon, 6 May 2024 16:13:06 +0800 Subject: [PATCH 002/402] [VL][CI] Fix nightly build job (#5562) adapt to the changes on duckdb from meta/velox --- .github/workflows/velox_nightly.yml | 2 +- cpp/velox/CMakeLists.txt | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/velox_nightly.yml b/.github/workflows/velox_nightly.yml index 78d4e53c399f..90073e8a3465 100644 --- a/.github/workflows/velox_nightly.yml +++ b/.github/workflows/velox_nightly.yml @@ -20,7 +20,7 @@ on: paths: - '.github/workflows/velox_nightly.yml' schedule: - - cron: '0 0 * * *' + - cron: '0 20 * * *' concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 55f45881b108..9ad14fdf7f9d 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -94,18 +94,7 @@ endmacro() macro(add_duckdb) find_package(DuckDB) if(NOT DuckDB_FOUND) - message(STATUS "Use duckdb provided by Velox.") - set(VELOX_BUILD_DUCKDB_PATH "${VELOX_BUILD_PATH}/_deps/duckdb-build") - add_velox_dependency(duckdb::static "${VELOX_BUILD_DUCKDB_PATH}/src/libduckdb_static.a") - add_velox_dependency(duckdb::fmt "${VELOX_BUILD_DUCKDB_PATH}/third_party/fmt/libduckdb_fmt.a") - add_velox_dependency(duckdb::query "${VELOX_BUILD_DUCKDB_PATH}/third_party/libpg_query/libduckdb_pg_query.a") - add_velox_dependency(duckdb::re2 "${VELOX_BUILD_DUCKDB_PATH}/third_party/re2/libduckdb_re2.a") - add_velox_dependency(duckdb::miniz "${VELOX_BUILD_DUCKDB_PATH}/third_party/miniz/libduckdb_miniz.a") - add_velox_dependency(duckdb::utf8 "${VELOX_BUILD_DUCKDB_PATH}/third_party/utf8proc/libduckdb_utf8proc.a") - add_velox_dependency(duckdb::hyperloglog "${VELOX_BUILD_DUCKDB_PATH}/third_party/hyperloglog/libduckdb_hyperloglog.a") - add_velox_dependency(duckdb::fastpforlib "${VELOX_BUILD_DUCKDB_PATH}/third_party/fastpforlib/libduckdb_fastpforlib.a") - add_velox_dependency(duckdb::mbedtls "${VELOX_BUILD_DUCKDB_PATH}/third_party/mbedtls/libduckdb_mbedtls.a") - add_velox_dependency(duckdb::fsst "${VELOX_BUILD_DUCKDB_PATH}/third_party/fsst/libduckdb_fsst.a") + message(FATAL_ERROR "Cannot find DuckDB.") else() message(STATUS "Found DuckDB library from ${DuckDB_DIR}") target_link_libraries(velox PUBLIC duckdb_static) From 7020ed3768ae1315481d6091a6aec33e3f93b66f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 6 May 2024 17:06:45 +0800 Subject: [PATCH 003/402] [VL] CI: Reformat gluten-it code with Spark331's scalafmt configuration (#5615) --- .../gluten/integration/tpc/Constants.scala | 28 ++-- .../gluten/integration/tpc/DataGen.scala | 43 +++--- .../gluten/integration/tpc/ShimUtils.scala | 12 +- .../gluten/integration/tpc/TpcRunner.scala | 25 ++- .../gluten/integration/tpc/TpcSuite.scala | 15 +- .../tpc/action/Parameterized.scala | 142 ++++++++---------- .../integration/tpc/action/Queries.scala | 72 ++++----- .../tpc/action/QueriesCompare.scala | 105 ++++++------- .../integration/tpc/ds/TpcdsDataGen.scala | 112 ++++++-------- .../integration/tpc/ds/TpcdsSuite.scala | 44 +++--- .../integration/tpc/h/TpchDataGen.scala | 141 +++++++---------- .../gluten/integration/tpc/h/TpchSuite.scala | 35 +++-- .../history/GlutenItHistoryServerPlugin.scala | 85 ++++++----- .../deploy/history/HistoryServerHelper.scala | 16 +- .../org/apache/spark/sql/ConfUtils.scala | 10 +- .../org/apache/spark/sql/QueryRunner.scala | 25 +-- .../org/apache/spark/sql/TestUtils.scala | 29 ++-- 17 files changed, 430 insertions(+), 509 deletions(-) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala index 7564f6dce90a..d39a16c325ef 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala @@ -18,7 +18,14 @@ package org.apache.gluten.integration.tpc import org.apache.spark.SparkConf import org.apache.spark.sql.TypeUtils -import org.apache.spark.sql.types.{DateType, DecimalType, DoubleType, IntegerType, LongType, StringType} +import org.apache.spark.sql.types.{ + DateType, + DecimalType, + DoubleType, + IntegerType, + LongType, + StringType +} import java.sql.Date @@ -33,16 +40,15 @@ object Constants { .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") .set("spark.sql.optimizer.runtime.bloomFilter.enabled", "true") .set("spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold", "0") - .set( - "spark.gluten.sql.columnar.physicalJoinOptimizeEnable", - "false" - ) // q72 slow if false, q64 fails if true + .set("spark.gluten.sql.columnar.physicalJoinOptimizeEnable", "false") // q72 slow if false, q64 fails if true val VELOX_WITH_CELEBORN_CONF: SparkConf = new SparkConf(false) .set("spark.gluten.sql.columnar.forceShuffledHashJoin", "true") .set("spark.sql.parquet.enableVectorizedReader", "true") .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set("spark.shuffle.manager", "org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager") + .set( + "spark.shuffle.manager", + "org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager") .set("spark.celeborn.shuffle.writer", "hash") .set("spark.celeborn.push.replicate.enabled", "false") .set("spark.celeborn.client.shuffle.compression.codec", "none") @@ -51,10 +57,7 @@ object Constants { .set("spark.dynamicAllocation.enabled", "false") .set("spark.sql.optimizer.runtime.bloomFilter.enabled", "true") .set("spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold", "0") - .set( - "spark.gluten.sql.columnar.physicalJoinOptimizeEnable", - "false" - ) // q72 slow if false, q64 fails if true + .set("spark.gluten.sql.columnar.physicalJoinOptimizeEnable", "false") // q72 slow if false, q64 fails if true .set("spark.celeborn.push.data.timeout", "600s") .set("spark.celeborn.push.limit.inFlight.timeout", "1200s") @@ -72,10 +75,7 @@ object Constants { .set("spark.dynamicAllocation.enabled", "false") .set("spark.sql.optimizer.runtime.bloomFilter.enabled", "true") .set("spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold", "0") - .set( - "spark.gluten.sql.columnar.physicalJoinOptimizeEnable", - "false" - ) + .set("spark.gluten.sql.columnar.physicalJoinOptimizeEnable", "false") @deprecated val TYPE_MODIFIER_DATE_AS_DOUBLE: TypeModifier = diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala index 5c092089def7..e810a4dc2316 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala @@ -23,7 +23,7 @@ trait DataGen { } abstract class TypeModifier(val predicate: DataType => Boolean, val to: DataType) - extends Serializable { + extends Serializable { def modValue(value: Any): Any } @@ -32,29 +32,30 @@ class NoopModifier(t: DataType) extends TypeModifier(_ => true, t) { } object DataGen { - def getRowModifier(schema: StructType, typeModifiers: List[TypeModifier]): Int => TypeModifier = { - val modifiers = schema.fields.map { - f => - val matchedModifiers = typeModifiers.flatMap { - m => - if (m.predicate.apply(f.dataType)) { - Some(m) - } else { - None - } - } - if (matchedModifiers.isEmpty) { - new NoopModifier(f.dataType) + def getRowModifier( + schema: StructType, + typeModifiers: List[TypeModifier]): Int => TypeModifier = { + val modifiers = schema.fields.map { f => + val matchedModifiers = typeModifiers.flatMap { m => + if (m.predicate.apply(f.dataType)) { + Some(m) } else { - if (matchedModifiers.size > 1) { - println( - s"More than one type modifiers specified for type ${f.dataType}, " + - s"use first one in the list") - } - matchedModifiers.head // use the first one that matches + None + } + } + if (matchedModifiers.isEmpty) { + new NoopModifier(f.dataType) + } else { + if (matchedModifiers.size > 1) { + println( + s"More than one type modifiers specified for type ${f.dataType}, " + + s"use first one in the list") } + matchedModifiers.head // use the first one that matches + } } - i => modifiers(i) + i => + modifiers(i) } def modifySchema(schema: StructType, rowModifier: Int => TypeModifier): StructType = { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala index c64fa160fd29..19e15df5cca7 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala @@ -25,13 +25,17 @@ object ShimUtils { def getExpressionEncoder(schema: StructType): ExpressionEncoder[Row] = { try { - RowEncoder.getClass.getMethod("apply", classOf[StructType]) - .invoke(RowEncoder, schema).asInstanceOf[ExpressionEncoder[Row]] + RowEncoder.getClass + .getMethod("apply", classOf[StructType]) + .invoke(RowEncoder, schema) + .asInstanceOf[ExpressionEncoder[Row]] } catch { case _: Exception => // to be compatible with Spark 3.5 and later - ExpressionEncoder.getClass.getMethod("apply", classOf[StructType]) - .invoke(ExpressionEncoder, schema).asInstanceOf[ExpressionEncoder[Row]] + ExpressionEncoder.getClass + .getMethod("apply", classOf[StructType]) + .invoke(ExpressionEncoder, schema) + .asInstanceOf[ExpressionEncoder[Row]] } } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala index ab76dc68cab8..908b8206eecd 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala @@ -48,20 +48,19 @@ class TpcRunner(val queryResourceFolder: String, val dataPath: String) { object TpcRunner { def createTables(spark: SparkSession, dataPath: String): Unit = { val files = new File(dataPath).listFiles() - files.foreach( - file => { - if (spark.catalog.tableExists(file.getName)) { - println("Table exists: " + file.getName) - } else { - println("Creating catalog table: " + file.getName) - spark.catalog.createTable(file.getName, file.getAbsolutePath, "parquet") - try { - spark.catalog.recoverPartitions(file.getName) - } catch { - case _: AnalysisException => - } + files.foreach(file => { + if (spark.catalog.tableExists(file.getName)) { + println("Table exists: " + file.getName) + } else { + println("Creating catalog table: " + file.getName) + spark.catalog.createTable(file.getName, file.getAbsolutePath, "parquet") + try { + spark.catalog.recoverPartitions(file.getName) + } catch { + case _: AnalysisException => } - }) + } + }) } private def delete(path: String): Unit = { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala index 058657976868..f7605e273eb1 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala @@ -68,7 +68,9 @@ abstract class TpcSuite( .setWarningOnOverriding("spark.executor.metrics.pollingInterval", "0") sessionSwitcher.defaultConf().setWarningOnOverriding("spark.network.timeout", "3601s") sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.broadcastTimeout", "1800") - sessionSwitcher.defaultConf().setWarningOnOverriding("spark.network.io.preferDirectBufs", "false") + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.network.io.preferDirectBufs", "false") sessionSwitcher .defaultConf() .setWarningOnOverriding("spark.unsafe.exceptionOnMemoryLeak", s"$errorOnMemLeak") @@ -113,8 +115,8 @@ abstract class TpcSuite( sessionSwitcher.defaultConf().setWarningOnOverriding("spark.default.parallelism", "1") } - extraSparkConf.toStream.foreach { - kv => sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2) + extraSparkConf.toStream.foreach { kv => + sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2) } // register sessions @@ -134,10 +136,9 @@ abstract class TpcSuite( } def run(): Boolean = { - val succeed = actions.forall { - action => - resetLogLevel() // to prevent log level from being set by unknown external codes - action.execute(this) + val succeed = actions.forall { action => + resetLogLevel() // to prevent log level from being set by unknown external codes + action.execute(this) } succeed } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala index f066659eff80..b4f7a53943e4 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala @@ -36,26 +36,22 @@ class Parameterized( configDimensions: Seq[Dim], excludedCombinations: Seq[Set[DimKv]], metrics: Array[String]) - extends Action { + extends Action { private def validateDims(configDimensions: Seq[Dim]): Unit = { - if ( - configDimensions - .map( - dim => { + if (configDimensions + .map(dim => { dim.name }) - .toSet - .size != configDimensions.size - ) { + .toSet + .size != configDimensions.size) { throw new IllegalStateException("Duplicated dimension name found") } - configDimensions.foreach { - dim => - if (dim.dimValues.map(dimValue => dimValue.name).toSet.size != dim.dimValues.size) { - throw new IllegalStateException("Duplicated dimension value found") - } + configDimensions.foreach { dim => + if (dim.dimValues.map(dimValue => dimValue.name).toSet.size != dim.dimValues.size) { + throw new IllegalStateException("Duplicated dimension value found") + } } } @@ -70,26 +66,23 @@ class Parameterized( intermediateConf: Seq[(String, String)]): Unit = { if (dimOffset == dimCount) { // we got one coordinate - excludedCombinations.foreach { - ec: Set[DimKv] => - if (ec.forall { - kv => + excludedCombinations.foreach { ec: Set[DimKv] => + if (ec.forall { kv => intermediateCoordinates.contains(kv.k) && intermediateCoordinates(kv.k) == kv.v - }) { - println(s"Coordinate ${Coordinate(intermediateCoordinates)} excluded by $ec.") - return - } + }) { + println(s"Coordinate ${Coordinate(intermediateCoordinates)} excluded by $ec.") + return + } } coordinateMap(Coordinate(intermediateCoordinates)) = intermediateConf return } val dim = configDimensions(dimOffset) - dim.dimValues.foreach { - dimValue => - fillCoordinates( - dimOffset + 1, - intermediateCoordinates + (dim.name -> dimValue.name), - intermediateConf ++ dimValue.conf) + dim.dimValues.foreach { dimValue => + fillCoordinates( + dimOffset + 1, + intermediateCoordinates + (dim.name -> dimValue.name), + intermediateConf ++ dimValue.conf) } } @@ -110,45 +103,40 @@ class Parameterized( case (c, idx) => println(s" $idx: $c") } - coordinates.foreach { - entry => - // register one session per coordinate - val coordinate = entry._1 - val coordinateConf = entry._2 - val conf = testConf.clone() - conf.setAllWarningOnOverriding(coordinateConf) - sessionSwitcher.registerSession(coordinate.toString, conf) + coordinates.foreach { entry => + // register one session per coordinate + val coordinate = entry._1 + val coordinateConf = entry._2 + val conf = testConf.clone() + conf.setAllWarningOnOverriding(coordinateConf) + sessionSwitcher.registerSession(coordinate.toString, conf) } val runQueryIds = queries.select(tpcSuite) // warm up - (0 until warmupIterations).foreach { - _ => - runQueryIds.foreach { - queryId => Parameterized.warmUp(queryId, tpcSuite.desc(), sessionSwitcher, runner) - } + (0 until warmupIterations).foreach { _ => + runQueryIds.foreach { queryId => + Parameterized.warmUp(queryId, tpcSuite.desc(), sessionSwitcher, runner) + } } - val results = coordinates.flatMap { - entry => - val coordinate = entry._1 - val coordinateResults = (0 until iterations).flatMap { - iteration => - println(s"Running tests (iteration $iteration) with coordinate $coordinate...") - runQueryIds.map { - queryId => - Parameterized.runTpcQuery( - runner, - sessionSwitcher, - queryId, - coordinate, - tpcSuite.desc(), - explain, - metrics) - } - }.toList - coordinateResults + val results = coordinates.flatMap { entry => + val coordinate = entry._1 + val coordinateResults = (0 until iterations).flatMap { iteration => + println(s"Running tests (iteration $iteration) with coordinate $coordinate...") + runQueryIds.map { queryId => + Parameterized.runTpcQuery( + runner, + sessionSwitcher, + queryId, + coordinate, + tpcSuite.desc(), + explain, + metrics) + } + }.toList + coordinateResults } val dimNames = configDimensions.map(dim => dim.name) @@ -164,8 +152,7 @@ class Parameterized( "RAM statistics: JVM Heap size: %d KiB (total %d KiB), Process RSS: %d KiB\n", RamStat.getJvmHeapUsed(), RamStat.getJvmHeapTotal(), - RamStat.getProcessRamUsed() - ) + RamStat.getProcessRamUsed()) println("") println("Test report: ") @@ -225,25 +212,22 @@ case class TestResultLines( fields.append("Row Count") fields.append("Query Time (Millis)") printf(fmt, fields: _*) - lines.foreach { - line => - val values = ArrayBuffer[Any](line.queryId, line.succeed) - dimNames.foreach { - dimName => - val coordinate = line.coordinate.coordinate - if (!coordinate.contains(dimName)) { - throw new IllegalStateException("Dimension name not found" + dimName) - } - values.append(coordinate(dimName)) - } - metricNames.foreach { - metricName => - val metrics = line.metrics - values.append(metrics.getOrElse(metricName, "N/A")) + lines.foreach { line => + val values = ArrayBuffer[Any](line.queryId, line.succeed) + dimNames.foreach { dimName => + val coordinate = line.coordinate.coordinate + if (!coordinate.contains(dimName)) { + throw new IllegalStateException("Dimension name not found" + dimName) } - values.append(line.rowCount.getOrElse("N/A")) - values.append(line.executionTimeMillis.getOrElse("N/A")) - printf(fmt, values: _*) + values.append(coordinate(dimName)) + } + metricNames.foreach { metricName => + val metrics = line.metrics + values.append(metrics.getOrElse(metricName, "N/A")) + } + values.append(line.rowCount.getOrElse("N/A")) + values.append(line.executionTimeMillis.getOrElse("N/A")) + printf(fmt, values: _*) } } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala index dc4ffe622895..c5f883189d29 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala @@ -27,24 +27,22 @@ case class Queries( explain: Boolean, iterations: Int, randomKillTasks: Boolean) - extends Action { + extends Action { override def execute(tpcSuite: TpcSuite): Boolean = { val runQueryIds = queries.select(tpcSuite) val runner: TpcRunner = new TpcRunner(tpcSuite.queryResource(), tpcSuite.dataWritePath(scale)) - val results = (0 until iterations).flatMap { - iteration => - println(s"Running tests (iteration $iteration)...") - runQueryIds.map { - queryId => - Queries.runTpcQuery( - runner, - tpcSuite.sessionSwitcher, - queryId, - tpcSuite.desc(), - explain, - randomKillTasks) - } + val results = (0 until iterations).flatMap { iteration => + println(s"Running tests (iteration $iteration)...") + runQueryIds.map { queryId => + Queries.runTpcQuery( + runner, + tpcSuite.sessionSwitcher, + queryId, + tpcSuite.desc(), + explain, + randomKillTasks) + } }.toList val passedCount = results.count(l => l.testPassed) @@ -58,8 +56,7 @@ case class Queries( "RAM statistics: JVM Heap size: %d KiB (total %d KiB), Process RSS: %d KiB\n", RamStat.getJvmHeapUsed(), RamStat.getJvmHeapTotal(), - RamStat.getProcessRamUsed() - ) + RamStat.getProcessRamUsed()) println("") println("Test report: ") @@ -112,17 +109,14 @@ object Queries { "Query ID", "Was Passed", "Row Count", - "Query Time (Millis)" - ) - results.foreach { - line => - printf( - "|%15s|%15s|%30s|%30s|\n", - line.queryId, - line.testPassed, - line.rowCount.getOrElse("N/A"), - line.executionTimeMillis.getOrElse("N/A") - ) + "Query Time (Millis)") + results.foreach { line => + printf( + "|%15s|%15s|%30s|%30s|\n", + line.queryId, + line.testPassed, + line.rowCount.getOrElse("N/A"), + line.executionTimeMillis.getOrElse("N/A")) } } @@ -131,19 +125,17 @@ object Queries { return Nil } List( - succeed.reduce( - (r1, r2) => - TestResultLine( - name, - testPassed = true, - if (r1.rowCount.nonEmpty && r2.rowCount.nonEmpty) - Some(r1.rowCount.get + r2.rowCount.get) - else None, - if (r1.executionTimeMillis.nonEmpty && r2.executionTimeMillis.nonEmpty) - Some(r1.executionTimeMillis.get + r2.executionTimeMillis.get) - else None, - None - ))) + succeed.reduce((r1, r2) => + TestResultLine( + name, + testPassed = true, + if (r1.rowCount.nonEmpty && r2.rowCount.nonEmpty) + Some(r1.rowCount.get + r2.rowCount.get) + else None, + if (r1.executionTimeMillis.nonEmpty && r2.executionTimeMillis.nonEmpty) + Some(r1.executionTimeMillis.get + r2.executionTimeMillis.get) + else None, + None))) } private def runTpcQuery( diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala index f841b58278dc..5e8e2d6136f7 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala @@ -27,23 +27,21 @@ case class QueriesCompare( queries: QuerySelector, explain: Boolean, iterations: Int) - extends Action { + extends Action { override def execute(tpcSuite: TpcSuite): Boolean = { val runner: TpcRunner = new TpcRunner(tpcSuite.queryResource(), tpcSuite.dataWritePath(scale)) val runQueryIds = queries.select(tpcSuite) - val results = (0 until iterations).flatMap { - iteration => - println(s"Running tests (iteration $iteration)...") - runQueryIds.map { - queryId => - QueriesCompare.runTpcQuery( - queryId, - explain, - tpcSuite.desc(), - tpcSuite.sessionSwitcher, - runner) - } + val results = (0 until iterations).flatMap { iteration => + println(s"Running tests (iteration $iteration)...") + runQueryIds.map { queryId => + QueriesCompare.runTpcQuery( + queryId, + explain, + tpcSuite.desc(), + tpcSuite.sessionSwitcher, + runner) + } }.toList val passedCount = results.count(l => l.testPassed) @@ -57,8 +55,7 @@ case class QueriesCompare( "RAM statistics: JVM Heap size: %d KiB (total %d KiB), Process RSS: %d KiB\n", RamStat.getJvmHeapUsed(), RamStat.getJvmHeapTotal(), - RamStat.getProcessRamUsed() - ) + RamStat.getProcessRamUsed()) println("") println("Test report: ") @@ -73,7 +70,8 @@ case class QueriesCompare( println("No failed queries. ") println("") } else { - println("Failed queries (a failed query with correct row count indicates value mismatches): ") + println( + "Failed queries (a failed query with correct row count indicates value mismatches): ") println("") QueriesCompare.printResults(results.filter(!_.testPassed)) println("") @@ -116,28 +114,23 @@ object QueriesCompare { "Actual Row Count", "Baseline Query Time (Millis)", "Query Time (Millis)", - "Query Time Variation" - ) - results.foreach { - line => - val timeVariation = - if ( - line.expectedExecutionTimeMillis.nonEmpty && line.actualExecutionTimeMillis.nonEmpty - ) { - Some( - ((line.expectedExecutionTimeMillis.get - line.actualExecutionTimeMillis.get).toDouble - / line.actualExecutionTimeMillis.get.toDouble) * 100) - } else None - printf( - "|%15s|%15s|%30s|%30s|%30s|%30s|%30s|\n", - line.queryId, - line.testPassed, - line.expectedRowCount.getOrElse("N/A"), - line.actualRowCount.getOrElse("N/A"), - line.expectedExecutionTimeMillis.getOrElse("N/A"), - line.actualExecutionTimeMillis.getOrElse("N/A"), - timeVariation.map("%15.2f%%".format(_)).getOrElse("N/A") - ) + "Query Time Variation") + results.foreach { line => + val timeVariation = + if (line.expectedExecutionTimeMillis.nonEmpty && line.actualExecutionTimeMillis.nonEmpty) { + Some( + ((line.expectedExecutionTimeMillis.get - line.actualExecutionTimeMillis.get).toDouble + / line.actualExecutionTimeMillis.get.toDouble) * 100) + } else None + printf( + "|%15s|%15s|%30s|%30s|%30s|%30s|%30s|\n", + line.queryId, + line.testPassed, + line.expectedRowCount.getOrElse("N/A"), + line.actualRowCount.getOrElse("N/A"), + line.expectedExecutionTimeMillis.getOrElse("N/A"), + line.actualExecutionTimeMillis.getOrElse("N/A"), + timeVariation.map("%15.2f%%".format(_)).getOrElse("N/A")) } } @@ -146,25 +139,23 @@ object QueriesCompare { return Nil } List( - succeed.reduce( - (r1, r2) => - TestResultLine( - name, - testPassed = true, - if (r1.expectedRowCount.nonEmpty && r2.expectedRowCount.nonEmpty) - Some(r1.expectedRowCount.get + r2.expectedRowCount.get) - else None, - if (r1.actualRowCount.nonEmpty && r2.actualRowCount.nonEmpty) - Some(r1.actualRowCount.get + r2.actualRowCount.get) - else None, - if (r1.expectedExecutionTimeMillis.nonEmpty && r2.expectedExecutionTimeMillis.nonEmpty) - Some(r1.expectedExecutionTimeMillis.get + r2.expectedExecutionTimeMillis.get) - else None, - if (r1.actualExecutionTimeMillis.nonEmpty && r2.actualExecutionTimeMillis.nonEmpty) - Some(r1.actualExecutionTimeMillis.get + r2.actualExecutionTimeMillis.get) - else None, - None - ))) + succeed.reduce((r1, r2) => + TestResultLine( + name, + testPassed = true, + if (r1.expectedRowCount.nonEmpty && r2.expectedRowCount.nonEmpty) + Some(r1.expectedRowCount.get + r2.expectedRowCount.get) + else None, + if (r1.actualRowCount.nonEmpty && r2.actualRowCount.nonEmpty) + Some(r1.actualRowCount.get + r2.actualRowCount.get) + else None, + if (r1.expectedExecutionTimeMillis.nonEmpty && r2.expectedExecutionTimeMillis.nonEmpty) + Some(r1.expectedExecutionTimeMillis.get + r2.expectedExecutionTimeMillis.get) + else None, + if (r1.actualExecutionTimeMillis.nonEmpty && r2.actualExecutionTimeMillis.nonEmpty) + Some(r1.actualExecutionTimeMillis.get + r2.actualExecutionTimeMillis.get) + else None, + None))) } private[tpc] def runTpcQuery( diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala index 081e547477f7..82d16dd90f1a 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala @@ -33,8 +33,8 @@ class TpcdsDataGen( dir: String, typeModifiers: List[TypeModifier] = List(), val genPartitionedData: Boolean) - extends Serializable - with DataGen { + extends Serializable + with DataGen { def writeParquetTable(t: Table): Unit = { val name = t.getName @@ -97,25 +97,23 @@ class TpcdsDataGen( val tablePath = dir + File.separator + tableName spark .range(0, partitions, 1L, partitions) - .mapPartitions { - itr => - val id = itr.toArray - if (id.length != 1) { - throw new IllegalStateException() - } - val options = new Options() - options.scale = scale - options.parallelism = partitions - val session = options.toSession - val chunkSession = session.withChunkNumber(id(0).toInt + 1) - val results = Results.constructResults(t, chunkSession).asScala.toIterator - results.map { - parentAndChildRow => - // Skip child table when generating parent table, - // we generate every table individually no matter it is parent or child. - val array: Array[String] = parentAndChildRow.get(0).asScala.toArray - Row(array: _*) - } + .mapPartitions { itr => + val id = itr.toArray + if (id.length != 1) { + throw new IllegalStateException() + } + val options = new Options() + options.scale = scale + options.parallelism = partitions + val session = options.toSession + val chunkSession = session.withChunkNumber(id(0).toInt + 1) + val results = Results.constructResults(t, chunkSession).asScala.toIterator + results.map { parentAndChildRow => + // Skip child table when generating parent table, + // we generate every table individually no matter it is parent or child. + val array: Array[String] = parentAndChildRow.get(0).asScala.toArray + Row(array: _*) + } }(ShimUtils.getExpressionEncoder(stringSchema)) .select(columns: _*) .write @@ -168,8 +166,7 @@ object TpcdsDataGen { StructField("cs_net_paid_inc_tax", DecimalType(7, 2)), StructField("cs_net_paid_inc_ship", DecimalType(7, 2)), StructField("cs_net_paid_inc_ship_tax", DecimalType(7, 2)), - StructField("cs_net_profit", DecimalType(7, 2)) - )) + StructField("cs_net_profit", DecimalType(7, 2)))) } private def catalogReturnsSchema = { @@ -201,8 +198,7 @@ object TpcdsDataGen { StructField("cr_refunded_cash", DecimalType(7, 2)), StructField("cr_reversed_charge", DecimalType(7, 2)), StructField("cr_store_credit", DecimalType(7, 2)), - StructField("cr_net_loss", DecimalType(7, 2)) - )) + StructField("cr_net_loss", DecimalType(7, 2)))) } private def inventorySchema = { @@ -211,8 +207,7 @@ object TpcdsDataGen { StructField("inv_date_sk", LongType), StructField("inv_item_sk", LongType), StructField("inv_warehouse_sk", LongType), - StructField("inv_quantity_on_hand", LongType) - )) + StructField("inv_quantity_on_hand", LongType))) } private def storeSalesSchema = { @@ -240,8 +235,7 @@ object TpcdsDataGen { StructField("ss_coupon_amt", DecimalType(7, 2)), StructField("ss_net_paid", DecimalType(7, 2)), StructField("ss_net_paid_inc_tax", DecimalType(7, 2)), - StructField("ss_net_profit", DecimalType(7, 2)) - )) + StructField("ss_net_profit", DecimalType(7, 2)))) } private def storeReturnsSchema = { @@ -266,8 +260,7 @@ object TpcdsDataGen { StructField("sr_refunded_cash", DecimalType(7, 2)), StructField("sr_reversed_charge", DecimalType(7, 2)), StructField("sr_store_credit", DecimalType(7, 2)), - StructField("sr_net_loss", DecimalType(7, 2)) - )) + StructField("sr_net_loss", DecimalType(7, 2)))) } private def webSalesSchema = { @@ -306,8 +299,7 @@ object TpcdsDataGen { StructField("ws_net_paid_inc_tax", DecimalType(7, 2)), StructField("ws_net_paid_inc_ship", DecimalType(7, 2)), StructField("ws_net_paid_inc_ship_tax", DecimalType(7, 2)), - StructField("ws_net_profit", DecimalType(7, 2)) - )) + StructField("ws_net_profit", DecimalType(7, 2)))) } private def webReturnsSchema = { @@ -336,8 +328,7 @@ object TpcdsDataGen { StructField("wr_refunded_cash", DecimalType(7, 2)), StructField("wr_reversed_charge", DecimalType(7, 2)), StructField("wr_account_credit", DecimalType(7, 2)), - StructField("wr_net_loss", DecimalType(7, 2)) - )) + StructField("wr_net_loss", DecimalType(7, 2)))) } private def callCenterSchema = { @@ -373,8 +364,7 @@ object TpcdsDataGen { StructField("cc_zip", StringType), StructField("cc_country", StringType), StructField("cc_gmt_offset", DecimalType(5, 2)), - StructField("cc_tax_percentage", DecimalType(5, 2)) - )) + StructField("cc_tax_percentage", DecimalType(5, 2)))) } private def catalogPageSchema = { @@ -388,8 +378,7 @@ object TpcdsDataGen { StructField("cp_catalog_number", LongType), StructField("cp_catalog_page_number", LongType), StructField("cp_description", StringType), - StructField("cp_type", StringType) - )) + StructField("cp_type", StringType))) } private def customerSchema = { @@ -412,8 +401,7 @@ object TpcdsDataGen { StructField("c_birth_country", StringType), StructField("c_login", StringType), StructField("c_email_address", StringType), - StructField("c_last_review_date", StringType) - )) + StructField("c_last_review_date", StringType))) } private def customerAddressSchema = { @@ -431,8 +419,7 @@ object TpcdsDataGen { StructField("ca_zip", StringType), StructField("ca_country", StringType), StructField("ca_gmt_offset", DecimalType(5, 2)), - StructField("ca_location_type", StringType) - )) + StructField("ca_location_type", StringType))) } private def customerDemographicsSchema = { @@ -446,8 +433,7 @@ object TpcdsDataGen { StructField("cd_credit_rating", StringType), StructField("cd_dep_count", LongType), StructField("cd_dep_employed_count", LongType), - StructField("cd_dep_college_count", LongType) - )) + StructField("cd_dep_college_count", LongType))) } private def dateDimSchema = { @@ -480,8 +466,7 @@ object TpcdsDataGen { StructField("d_current_week", StringType), StructField("d_current_month", StringType), StructField("d_current_quarter", StringType), - StructField("d_current_year", StringType) - )) + StructField("d_current_year", StringType))) } private def householdDemographicsSchema = { @@ -491,8 +476,7 @@ object TpcdsDataGen { StructField("hd_income_band_sk", LongType), StructField("hd_buy_potential", StringType), StructField("hd_dep_count", LongType), - StructField("hd_vehicle_count", LongType) - )) + StructField("hd_vehicle_count", LongType))) } private def incomeBandSchema = { @@ -500,8 +484,7 @@ object TpcdsDataGen { Seq( StructField("ib_income_band_sk", LongType), StructField("ib_lower_bound", LongType), - StructField("ib_upper_bound", LongType) - )) + StructField("ib_upper_bound", LongType))) } private def itemSchema = { @@ -528,8 +511,7 @@ object TpcdsDataGen { StructField("i_units", StringType), StructField("i_container", StringType), StructField("i_manager_id", LongType), - StructField("i_product_name", StringType) - )) + StructField("i_product_name", StringType))) } private def promotionSchema = { @@ -553,8 +535,7 @@ object TpcdsDataGen { StructField("p_channel_demo", StringType), StructField("p_channel_details", StringType), StructField("p_purpose", StringType), - StructField("p_discount_active", StringType) - )) + StructField("p_discount_active", StringType))) } private def reasonSchema = { @@ -562,8 +543,7 @@ object TpcdsDataGen { Seq( StructField("r_reason_sk", LongType), StructField("r_reason_id", StringType), - StructField("r_reason_desc", StringType) - )) + StructField("r_reason_desc", StringType))) } private def shipModeSchema = { @@ -574,8 +554,7 @@ object TpcdsDataGen { StructField("sm_type", StringType), StructField("sm_code", StringType), StructField("sm_carrier", StringType), - StructField("sm_contract", StringType) - )) + StructField("sm_contract", StringType))) } private def storeSchema = { @@ -609,8 +588,7 @@ object TpcdsDataGen { StructField("s_zip", StringType), StructField("s_country", StringType), StructField("s_gmt_offset", DecimalType(5, 2)), - StructField("s_tax_precentage", DecimalType(5, 2)) - )) + StructField("s_tax_precentage", DecimalType(5, 2)))) } private def timeDimSchema = { @@ -625,8 +603,7 @@ object TpcdsDataGen { StructField("t_am_pm", StringType), StructField("t_shift", StringType), StructField("t_sub_shift", StringType), - StructField("t_meal_time", StringType) - )) + StructField("t_meal_time", StringType))) } private def warehouseSchema = { @@ -645,8 +622,7 @@ object TpcdsDataGen { StructField("w_state", StringType), StructField("w_zip", StringType), StructField("w_country", StringType), - StructField("w_gmt_offset", DecimalType(5, 2)) - )) + StructField("w_gmt_offset", DecimalType(5, 2)))) } private def webPageSchema = { @@ -665,8 +641,7 @@ object TpcdsDataGen { StructField("wp_char_count", LongType), StructField("wp_link_count", LongType), StructField("wp_image_count", LongType), - StructField("wp_max_ad_count", LongType) - )) + StructField("wp_max_ad_count", LongType))) } private def webSiteSchema = { @@ -697,7 +672,6 @@ object TpcdsDataGen { StructField("web_zip", StringType), StructField("web_country", StringType), StructField("web_gmt_offset", StringType), - StructField("web_tax_percentage", DecimalType(5, 2)) - )) + StructField("web_tax_percentage", DecimalType(5, 2)))) } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala index 37a88d446f3e..c703821c1b0f 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala @@ -18,7 +18,11 @@ package org.apache.gluten.integration.tpc.ds import org.apache.gluten.integration.tpc.{Constants, DataGen, TpcSuite, TypeModifier} import org.apache.gluten.integration.tpc.action.Action -import org.apache.gluten.integration.tpc.ds.TpcdsSuite.{ALL_QUERY_IDS, HISTORY_WRITE_PATH, TPCDS_WRITE_PATH} +import org.apache.gluten.integration.tpc.ds.TpcdsSuite.{ + ALL_QUERY_IDS, + HISTORY_WRITE_PATH, + TPCDS_WRITE_PATH +} import org.apache.spark.SparkConf @@ -41,24 +45,23 @@ class TpcdsSuite( val disableWscg: Boolean, val shufflePartitions: Int, val minimumScanPartitions: Boolean) - extends TpcSuite( - masterUrl, - actions, - testConf, - baselineConf, - extraSparkConf, - logLevel, - errorOnMemLeak, - enableUi, - enableHsUi, - hsUiPort, - offHeapSize, - disableAqe, - disableBhj, - disableWscg, - shufflePartitions, - minimumScanPartitions - ) { + extends TpcSuite( + masterUrl, + actions, + testConf, + baselineConf, + extraSparkConf, + logLevel, + errorOnMemLeak, + enableUi, + enableHsUi, + hsUiPort, + offHeapSize, + disableAqe, + disableBhj, + disableWscg, + shufflePartitions, + minimumScanPartitions) { override protected def historyWritePath(): String = HISTORY_WRITE_PATH @@ -191,7 +194,6 @@ object TpcdsSuite { "q96", "q97", "q98", - "q99" - ) + "q99") private val HISTORY_WRITE_PATH = "/tmp/tpcds-history" } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala index 18c557045d75..fa574f59c5d2 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala @@ -34,8 +34,8 @@ class TpchDataGen( partitions: Int, path: String, typeModifiers: List[TypeModifier] = List()) - extends Serializable - with DataGen { + extends Serializable + with DataGen { override def gen(): Unit = { generate(path, "lineitem", lineItemSchema, partitions, lineItemGenerator, lineItemParser) @@ -55,8 +55,8 @@ class TpchDataGen( } // lineitem - private def lineItemGenerator = { - (part: Int, partCount: Int) => new LineItemGenerator(scale, part, partCount) + private def lineItemGenerator = { (part: Int, partCount: Int) => + new LineItemGenerator(scale, part, partCount) } private def lineItemSchema = { @@ -77,8 +77,7 @@ class TpchDataGen( StructField("l_shipinstruct", StringType), StructField("l_shipmode", StringType), StructField("l_comment", StringType), - StructField("l_shipdate", DateType) - )) + StructField("l_shipdate", DateType))) } private def lineItemParser: LineItem => Row = @@ -99,12 +98,11 @@ class TpchDataGen( lineItem.getShipInstructions, lineItem.getShipMode, lineItem.getComment, - Date.valueOf(GenerateUtils.formatDate(lineItem.getShipDate)) - ) + Date.valueOf(GenerateUtils.formatDate(lineItem.getShipDate))) // customer - private def customerGenerator = { - (part: Int, partCount: Int) => new CustomerGenerator(scale, part, partCount) + private def customerGenerator = { (part: Int, partCount: Int) => + new CustomerGenerator(scale, part, partCount) } private def customerSchema = { @@ -117,8 +115,7 @@ class TpchDataGen( StructField("c_phone", StringType), StructField("c_acctbal", DecimalType(12, 2)), StructField("c_comment", StringType), - StructField("c_mktsegment", StringType) - )) + StructField("c_mktsegment", StringType))) } private def customerParser: Customer => Row = @@ -131,12 +128,11 @@ class TpchDataGen( customer.getPhone, BigDecimal.valueOf(customer.getAccountBalance), customer.getComment, - customer.getMarketSegment - ) + customer.getMarketSegment) // orders - private def orderGenerator = { - (part: Int, partCount: Int) => new OrderGenerator(scale, part, partCount) + private def orderGenerator = { (part: Int, partCount: Int) => + new OrderGenerator(scale, part, partCount) } private def orderSchema = { @@ -150,8 +146,7 @@ class TpchDataGen( StructField("o_clerk", StringType), StructField("o_shippriority", IntegerType), StructField("o_comment", StringType), - StructField("o_orderdate", DateType) - )) + StructField("o_orderdate", DateType))) } private def orderParser: Order => Row = @@ -165,12 +160,11 @@ class TpchDataGen( order.getClerk, order.getShipPriority, order.getComment, - Date.valueOf(GenerateUtils.formatDate(order.getOrderDate)) - ) + Date.valueOf(GenerateUtils.formatDate(order.getOrderDate))) // partsupp - private def partSupplierGenerator = { - (part: Int, partCount: Int) => new PartSupplierGenerator(scale, part, partCount) + private def partSupplierGenerator = { (part: Int, partCount: Int) => + new PartSupplierGenerator(scale, part, partCount) } private def partSupplierSchema = { @@ -180,8 +174,7 @@ class TpchDataGen( StructField("ps_suppkey", LongType), StructField("ps_availqty", IntegerType), StructField("ps_supplycost", DecimalType(12, 2)), - StructField("ps_comment", StringType) - )) + StructField("ps_comment", StringType))) } private def partSupplierParser: PartSupplier => Row = @@ -191,12 +184,11 @@ class TpchDataGen( ps.getSupplierKey, ps.getAvailableQuantity, BigDecimal.valueOf(ps.getSupplyCost), - ps.getComment - ) + ps.getComment) // supplier - private def supplierGenerator = { - (part: Int, partCount: Int) => new SupplierGenerator(scale, part, partCount) + private def supplierGenerator = { (part: Int, partCount: Int) => + new SupplierGenerator(scale, part, partCount) } private def supplierSchema = { @@ -208,8 +200,7 @@ class TpchDataGen( StructField("s_nationkey", LongType), StructField("s_phone", StringType), StructField("s_acctbal", DecimalType(12, 2)), - StructField("s_comment", StringType) - )) + StructField("s_comment", StringType))) } private def supplierParser: Supplier => Row = @@ -221,11 +212,12 @@ class TpchDataGen( s.getNationKey, s.getPhone, BigDecimal.valueOf(s.getAccountBalance), - s.getComment - ) + s.getComment) // nation - private def nationGenerator = { () => new NationGenerator() } + private def nationGenerator = { () => + new NationGenerator() + } private def nationSchema = { StructType( @@ -233,22 +225,15 @@ class TpchDataGen( StructField("n_nationkey", LongType), StructField("n_name", StringType), StructField("n_regionkey", LongType), - StructField("n_comment", StringType) - )) + StructField("n_comment", StringType))) } private def nationParser: Nation => Row = - nation => - Row( - nation.getNationKey, - nation.getName, - nation.getRegionKey, - nation.getComment - ) + nation => Row(nation.getNationKey, nation.getName, nation.getRegionKey, nation.getComment) // part - private def partGenerator = { - (part: Int, partCount: Int) => new PartGenerator(scale, part, partCount) + private def partGenerator = { (part: Int, partCount: Int) => + new PartGenerator(scale, part, partCount) } private def partSchema = { @@ -262,8 +247,7 @@ class TpchDataGen( StructField("p_container", StringType), StructField("p_retailprice", DecimalType(12, 2)), StructField("p_comment", StringType), - StructField("p_brand", StringType) - )) + StructField("p_brand", StringType))) } private def partParser: Part => Row = @@ -277,28 +261,23 @@ class TpchDataGen( part.getContainer, BigDecimal.valueOf(part.getRetailPrice), part.getComment, - part.getBrand - ) + part.getBrand) // region - private def regionGenerator = { () => new RegionGenerator() } + private def regionGenerator = { () => + new RegionGenerator() + } private def regionSchema = { StructType( Seq( StructField("r_regionkey", LongType), StructField("r_name", StringType), - StructField("r_comment", StringType) - )) + StructField("r_comment", StringType))) } private def regionParser: Region => Row = - region => - Row( - region.getRegionKey, - region.getName, - region.getComment - ) + region => Row(region.getRegionKey, region.getName, region.getComment) // gen tpc-h data private def generate[U]( @@ -307,15 +286,9 @@ class TpchDataGen( schema: StructType, gen: () => java.lang.Iterable[U], parser: U => Row): Unit = { - generate( - dir, - tableName, - schema, - 1, - (_: Int, _: Int) => { - gen.apply() - }, - parser) + generate(dir, tableName, schema, 1, (_: Int, _: Int) => { + gen.apply() + }, parser) } private def generate[U]( @@ -330,25 +303,23 @@ class TpchDataGen( val modifiedSchema = DataGen.modifySchema(schema, rowModifier) spark .range(0, partitions, 1L, partitions) - .mapPartitions { - itr => - val id = itr.toArray - if (id.length != 1) { - throw new IllegalStateException() - } - val data = gen.apply(id(0).toInt + 1, partitions) - val dataItr = data.iterator() - val rows = dataItr.asScala.map { - item => - val row = parser(item) - val modifiedRow = Row(row.toSeq.zipWithIndex.map { - case (v, i) => - val modifier = rowModifier.apply(i) - modifier.modValue(v) - }.toArray: _*) - modifiedRow - } - rows + .mapPartitions { itr => + val id = itr.toArray + if (id.length != 1) { + throw new IllegalStateException() + } + val data = gen.apply(id(0).toInt + 1, partitions) + val dataItr = data.iterator() + val rows = dataItr.asScala.map { item => + val row = parser(item) + val modifiedRow = Row(row.toSeq.zipWithIndex.map { + case (v, i) => + val modifier = rowModifier.apply(i) + modifier.modValue(v) + }.toArray: _*) + modifiedRow + } + rows }(ShimUtils.getExpressionEncoder(modifiedSchema)) .write .mode(SaveMode.Overwrite) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala index 418c7ca6a08f..9fbd83dc2f66 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala @@ -41,24 +41,23 @@ class TpchSuite( val disableWscg: Boolean, val shufflePartitions: Int, val minimumScanPartitions: Boolean) - extends TpcSuite( - masterUrl, - actions, - testConf, - baselineConf, - extraSparkConf, - logLevel, - errorOnMemLeak, - enableUi, - enableHsUi, - hsUiPort, - offHeapSize, - disableAqe, - disableBhj, - disableWscg, - shufflePartitions, - minimumScanPartitions - ) { + extends TpcSuite( + masterUrl, + actions, + testConf, + baselineConf, + extraSparkConf, + logLevel, + errorOnMemLeak, + enableUi, + enableHsUi, + hsUiPort, + offHeapSize, + disableAqe, + disableBhj, + disableWscg, + shufflePartitions, + minimumScanPartitions) { override protected def historyWritePath(): String = HISTORY_WRITE_PATH diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/GlutenItHistoryServerPlugin.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/GlutenItHistoryServerPlugin.scala index 33500c3e18c7..4720d3e4a8c0 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/GlutenItHistoryServerPlugin.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/GlutenItHistoryServerPlugin.scala @@ -20,7 +20,11 @@ import org.apache.spark.SparkConf import org.apache.spark.deploy.history.HistoryServerHelper.LogServerRpcEnvs import org.apache.spark.scheduler.SparkListener import org.apache.spark.sql.ConfUtils.ConfImplicits.SparkConfWrapper -import org.apache.spark.status.{AppHistoryServerPlugin, ElementTrackingStore, ExecutorSummaryWrapper} +import org.apache.spark.status.{ + AppHistoryServerPlugin, + ElementTrackingStore, + ExecutorSummaryWrapper +} import org.apache.spark.status.api.v1 import com.google.common.base.Preconditions @@ -60,52 +64,53 @@ class GlutenItHistoryServerPlugin extends AppHistoryServerPlugin { } } - override def createListeners(conf: SparkConf, store: ElementTrackingStore): Seq[SparkListener] = { + override def createListeners( + conf: SparkConf, + store: ElementTrackingStore): Seq[SparkListener] = { store.onFlush { val wrappers = org.apache.spark.util.Utils .tryWithResource(store.view(classOf[ExecutorSummaryWrapper]).closeableIterator()) { - iter => iter.asScala.toList + iter => + iter.asScala.toList } // create new executor summaries wrappers - .map { - wrapper => - Preconditions.checkArgument(wrapper.info.attributes.isEmpty) - new ExecutorSummaryWrapper( - new v1.ExecutorSummary( - id = wrapper.info.id, - hostPort = wrapper.info.hostPort, - isActive = wrapper.info.isActive, - rddBlocks = wrapper.info.rddBlocks, - memoryUsed = wrapper.info.memoryUsed, - diskUsed = wrapper.info.diskUsed, - totalCores = wrapper.info.totalCores, - maxTasks = wrapper.info.maxTasks, - activeTasks = wrapper.info.activeTasks, - failedTasks = wrapper.info.failedTasks, - completedTasks = wrapper.info.completedTasks, - totalTasks = wrapper.info.totalTasks, - totalDuration = wrapper.info.totalDuration, - totalGCTime = wrapper.info.totalGCTime, - totalInputBytes = wrapper.info.totalInputBytes, - totalShuffleRead = wrapper.info.totalShuffleRead, - totalShuffleWrite = wrapper.info.totalShuffleWrite, - isBlacklisted = wrapper.info.isBlacklisted, - maxMemory = wrapper.info.maxMemory, - addTime = wrapper.info.addTime, - removeTime = wrapper.info.removeTime, - removeReason = wrapper.info.removeReason, - executorLogs = rewriteLogs(wrapper.info.executorLogs, logServerRpcEnvs), - memoryMetrics = wrapper.info.memoryMetrics, - blacklistedInStages = wrapper.info.blacklistedInStages, - peakMemoryMetrics = wrapper.info.peakMemoryMetrics, - attributes = wrapper.info.attributes, - resources = wrapper.info.resources, - resourceProfileId = wrapper.info.resourceProfileId, - isExcluded = wrapper.info.isExcluded, - excludedInStages = wrapper.info.excludedInStages - )) + .map { wrapper => + Preconditions.checkArgument(wrapper.info.attributes.isEmpty) + new ExecutorSummaryWrapper( + new v1.ExecutorSummary( + id = wrapper.info.id, + hostPort = wrapper.info.hostPort, + isActive = wrapper.info.isActive, + rddBlocks = wrapper.info.rddBlocks, + memoryUsed = wrapper.info.memoryUsed, + diskUsed = wrapper.info.diskUsed, + totalCores = wrapper.info.totalCores, + maxTasks = wrapper.info.maxTasks, + activeTasks = wrapper.info.activeTasks, + failedTasks = wrapper.info.failedTasks, + completedTasks = wrapper.info.completedTasks, + totalTasks = wrapper.info.totalTasks, + totalDuration = wrapper.info.totalDuration, + totalGCTime = wrapper.info.totalGCTime, + totalInputBytes = wrapper.info.totalInputBytes, + totalShuffleRead = wrapper.info.totalShuffleRead, + totalShuffleWrite = wrapper.info.totalShuffleWrite, + isBlacklisted = wrapper.info.isBlacklisted, + maxMemory = wrapper.info.maxMemory, + addTime = wrapper.info.addTime, + removeTime = wrapper.info.removeTime, + removeReason = wrapper.info.removeReason, + executorLogs = rewriteLogs(wrapper.info.executorLogs, logServerRpcEnvs), + memoryMetrics = wrapper.info.memoryMetrics, + blacklistedInStages = wrapper.info.blacklistedInStages, + peakMemoryMetrics = wrapper.info.peakMemoryMetrics, + attributes = wrapper.info.attributes, + resources = wrapper.info.resources, + resourceProfileId = wrapper.info.resourceProfileId, + isExcluded = wrapper.info.isExcluded, + excludedInStages = wrapper.info.excludedInStages)) } .foreach(store.write(_)) } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/HistoryServerHelper.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/HistoryServerHelper.scala index fb991cb47691..649ef130ddad 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/HistoryServerHelper.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/deploy/history/HistoryServerHelper.scala @@ -46,10 +46,9 @@ object HistoryServerHelper { private def findFreePort(): Int = { val port = org.apache.spark.util.Utils - .tryWithResource(new ServerSocket(0)) { - socket => - socket.setReuseAddress(true) - socket.getLocalPort + .tryWithResource(new ServerSocket(0)) { socket => + socket.setReuseAddress(true) + socket.getLocalPort } if (port > 0) { return port @@ -80,11 +79,10 @@ object HistoryServerHelper { conf, conf.get(org.apache.spark.internal.config.Worker.SPARK_WORKER_RESOURCE_FILE)) - ShutdownHookManager.addShutdownHook( - () => { - workerRpcEnv.shutdown() - rpcEnv.shutdown() - }) + ShutdownHookManager.addShutdownHook(() => { + workerRpcEnv.shutdown() + rpcEnv.shutdown() + }) LogServerRpcEnvs(rpcEnv, workerRpcEnv, webUiPort, workerWebUiPort) } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/ConfUtils.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/ConfUtils.scala index 966d4624779d..66eb4a48f682 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/ConfUtils.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/ConfUtils.scala @@ -30,17 +30,15 @@ object ConfUtils { onOverriding => { Console.err.println( s"Overriding SparkConf key ${onOverriding.key}, old value: ${onOverriding.value}, new value: ${onOverriding.newValue}. ") - } - ) + }) } def setAllWarningOnOverriding(others: Iterable[(String, String)]): SparkConf = { var tmp: SparkConf = conf - others.foreach( - c => { - tmp = new SparkConfWrapper(tmp).setWarningOnOverriding(c._1, c._2) - }) + others.foreach(c => { + tmp = new SparkConfWrapper(tmp).setWarningOnOverriding(c._1, c._2) + }) tmp } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala index a4044c925a31..332e56043c45 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala @@ -18,7 +18,12 @@ package org.apache.spark.sql import org.apache.spark.{SparkContext, Success, TaskKilled} import org.apache.spark.executor.ExecutorMetrics -import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorMetricsUpdate, SparkListenerTaskEnd, SparkListenerTaskStart} +import org.apache.spark.scheduler.{ + SparkListener, + SparkListenerExecutorMetricsUpdate, + SparkListenerTaskEnd, + SparkListenerTaskStart +} import org.apache.spark.sql.KillTaskListener.INIT_WAIT_TIME_MS import com.google.common.base.Preconditions @@ -45,8 +50,7 @@ object QueryRunner { "ProcessTreePythonVMemory", "ProcessTreePythonRSSMemory", "ProcessTreeOtherVMemory", - "ProcessTreeOtherRSSMemory" - ) + "ProcessTreeOtherRSSMemory") def runTpcQuery( spark: SparkSession, @@ -90,12 +94,11 @@ object QueryRunner { RunResult(rows, millis, collectedMetrics) } finally { sc.removeSparkListener(metricsListener) - killTaskListener.foreach( - l => { - sc.removeSparkListener(l) - println(s"Successful kill rate ${"%.2f%%".format( - 100 * l.successfulKillRate())} during execution of app: ${sc.applicationId}") - }) + killTaskListener.foreach(l => { + sc.removeSparkListener(l) + println(s"Successful kill rate ${"%.2f%%" + .format(100 * l.successfulKillRate())} during execution of app: ${sc.applicationId}") + }) sc.setJobDescription(null) } } @@ -156,8 +159,8 @@ class KillTaskListener(val sc: SparkContext) extends SparkListener { sync.synchronized { val total = Math.min( stageKillMaxWaitTimeLookup.computeIfAbsent(taskStart.stageId, _ => Long.MaxValue), - stageKillWaitTimeLookup.computeIfAbsent(taskStart.stageId, _ => INIT_WAIT_TIME_MS) - ) + stageKillWaitTimeLookup + .computeIfAbsent(taskStart.stageId, _ => INIT_WAIT_TIME_MS)) val elapsed = System.currentTimeMillis() - startMs val remaining = total - elapsed if (remaining <= 0L) { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/TestUtils.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/TestUtils.scala index 03e70fbcc75c..c5af0e9b43ca 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/TestUtils.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/TestUtils.scala @@ -46,7 +46,8 @@ object TestUtils { override def toString: String = java.lang.Float.toString(value) // unsupported - override def compareTo(anotherFloat: FuzzyFloat): Int = throw new UnsupportedOperationException + override def compareTo(anotherFloat: FuzzyFloat): Int = + throw new UnsupportedOperationException override def hashCode(): Int = throw new UnsupportedOperationException } @@ -62,15 +63,14 @@ object TestUtils { // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for // equality test. // This function is copied from Catalyst's QueryTest - val converted: Seq[Row] = answer.map { - s => - Row.fromSeq(s.toSeq.map { - case d: java.math.BigDecimal => BigDecimal(d) - case b: Array[Byte] => b.toSeq - case f: Float => new FuzzyFloat(f) - case db: Double => new FuzzyDouble(db) - case o => o - }) + val converted: Seq[Row] = answer.map { s => + Row.fromSeq(s.toSeq.map { + case d: java.math.BigDecimal => BigDecimal(d) + case b: Array[Byte] => b.toSeq + case f: Float => new FuzzyFloat(f) + case db: Double => new FuzzyDouble(db) + case o => o + }) } if (sort) { converted.sortBy(_.toString()) @@ -83,11 +83,10 @@ object TestUtils { s""" | == Results == | ${sideBySide( - s"== Expected Answer - ${expectedAnswer.size} ==" +: - prepareAnswer(expectedAnswer).map(_.toString()), - s"== Actual Answer - ${sparkAnswer.size} ==" +: - prepareAnswer(sparkAnswer).map(_.toString()) - ).mkString("\n")} + s"== Expected Answer - ${expectedAnswer.size} ==" +: + prepareAnswer(expectedAnswer).map(_.toString()), + s"== Actual Answer - ${sparkAnswer.size} ==" +: + prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n")} """.stripMargin Some(errorMessage) } else { From 72e787c6470fa5abd733b3f65a90cc43e890367c Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Mon, 6 May 2024 20:44:41 +0800 Subject: [PATCH 004/402] [GLUTEN-5603] Add new added Spark3.4 UTs in Gluten (#5604) In Spark 3.4, there are some new added UTs vs Spark3.3 which haven't been added in Gluten yet. In this PR, we ported the ones under spark/sql but excluding folders like streaming, execution/command, execution/benchmark. --- .../utils/velox/VeloxTestSettings.scala | 19 +++++++++++++-- .../sql/GlutenDataFrameToSchemaSuite.scala | 19 +++++++++++++++ .../spark/sql/GlutenDatasetUnpivotSuite.scala | 19 +++++++++++++++ .../sql/GlutenLateralColumnAliasSuite.scala | 19 +++++++++++++++ .../spark/sql/GlutenParametersSuite.scala | 19 +++++++++++++++ .../GlutenResolveDefaultColumnsSuite.scala | 21 +++++++++++++++++ .../GlutenSubqueryHintPropagationSuite.scala | 21 +++++++++++++++++ .../spark/sql/GlutenUrlFunctionsSuite.scala | 19 +++++++++++++++ ...GlutenDeltaBasedDeleteFromTableSuite.scala | 23 +++++++++++++++++++ ...GlutenGroupBasedDeleteFromTableSuite.scala | 23 +++++++++++++++++++ ...lutenFileMetadataStructRowIndexSuite.scala | 23 +++++++++++++++++++ .../parquet/GlutenParquetRowIndexSuite.scala | 21 +++++++++++++++++ 12 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructRowIndexSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 8fcdf255fd69..033e084b8522 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -20,11 +20,11 @@ import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} -import org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite, GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite, GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter, GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite, GlutenDeleteFromTableSuite, GlutenFileDataSourceV2FallBackSuite, GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, GlutenMetadataColumnSuite, GlutenSupportsCatalogOptionsSuite, GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite} +import org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite, GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite, GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter, GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite, GlutenDeleteFromTableSuite, GlutenDeltaBasedDeleteFromTableSuite, GlutenFileDataSourceV2FallBackSuite, GlutenGroupBasedDeleteFromTableSuite, GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, GlutenMetadataColumnSuite, GlutenSupportsCatalogOptionsSuite, GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite} import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} import org.apache.spark.sql.execution.{FallbackStrategiesSuite, GlutenBroadcastExchangeSuite, GlutenCoalesceShufflePartitionsSuite, GlutenExchangeSuite, GlutenLocalBroadcastExchangeSuite, GlutenReplaceHashWithSortAggSuite, GlutenReuseExchangeAndSubquerySuite, GlutenSameResultSuite, GlutenSortSuite, GlutenSQLAggregateFunctionSuite, GlutenSQLWindowFunctionSuite, GlutenTakeOrderedAndProjectSuite} import org.apache.spark.sql.execution.adaptive.velox.VeloxAdaptiveQueryExecSuite -import org.apache.spark.sql.execution.datasources.{GlutenBucketingUtilsSuite, GlutenCSVReadSchemaSuite, GlutenDataSourceStrategySuite, GlutenDataSourceSuite, GlutenFileFormatWriterSuite, GlutenFileIndexSuite, GlutenFileMetadataStructSuite, GlutenFileSourceStrategySuite, GlutenHadoopFileLinesReaderSuite, GlutenHeaderCSVReadSchemaSuite, GlutenJsonReadSchemaSuite, GlutenMergedOrcReadSchemaSuite, GlutenMergedParquetReadSchemaSuite, GlutenOrcCodecSuite, GlutenOrcReadSchemaSuite, GlutenOrcV1AggregatePushDownSuite, GlutenOrcV2AggregatePushDownSuite, GlutenParquetCodecSuite, GlutenParquetReadSchemaSuite, GlutenParquetV1AggregatePushDownSuite, GlutenParquetV2AggregatePushDownSuite, GlutenPathFilterStrategySuite, GlutenPathFilterSuite, GlutenPruneFileSourcePartitionsSuite, GlutenV1WriteCommandSuite, GlutenVectorizedOrcReadSchemaSuite, GlutenVectorizedParquetReadSchemaSuite} +import org.apache.spark.sql.execution.datasources.{GlutenBucketingUtilsSuite, GlutenCSVReadSchemaSuite, GlutenDataSourceStrategySuite, GlutenDataSourceSuite, GlutenFileFormatWriterSuite, GlutenFileIndexSuite, GlutenFileMetadataStructRowIndexSuite, GlutenFileMetadataStructSuite, GlutenFileSourceStrategySuite, GlutenHadoopFileLinesReaderSuite, GlutenHeaderCSVReadSchemaSuite, GlutenJsonReadSchemaSuite, GlutenMergedOrcReadSchemaSuite, GlutenMergedParquetReadSchemaSuite, GlutenOrcCodecSuite, GlutenOrcReadSchemaSuite, GlutenOrcV1AggregatePushDownSuite, GlutenOrcV2AggregatePushDownSuite, GlutenParquetCodecSuite, GlutenParquetReadSchemaSuite, GlutenParquetV1AggregatePushDownSuite, GlutenParquetV2AggregatePushDownSuite, GlutenPathFilterStrategySuite, GlutenPathFilterSuite, GlutenPruneFileSourcePartitionsSuite, GlutenV1WriteCommandSuite, GlutenVectorizedOrcReadSchemaSuite, GlutenVectorizedParquetReadSchemaSuite} import org.apache.spark.sql.execution.datasources.binaryfile.GlutenBinaryFileFormatSuite import org.apache.spark.sql.execution.datasources.csv.{GlutenCSVLegacyTimeParserSuite, GlutenCSVv1Suite, GlutenCSVv2Suite} import org.apache.spark.sql.execution.datasources.exchange.GlutenValidateRequirementsSuite @@ -1151,6 +1151,21 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenHiveSQLQuerySuite] enableSuite[GlutenCollapseProjectExecTransformerSuite] enableSuite[GlutenSparkSessionExtensionSuite] + enableSuite[GlutenGroupBasedDeleteFromTableSuite] + enableSuite[GlutenDeltaBasedDeleteFromTableSuite] + enableSuite[GlutenDataFrameToSchemaSuite] + enableSuite[GlutenDatasetUnpivotSuite] + enableSuite[GlutenLateralColumnAliasSuite] + enableSuite[GlutenParametersSuite] + enableSuite[GlutenResolveDefaultColumnsSuite] + enableSuite[GlutenSubqueryHintPropagationSuite] + enableSuite[GlutenUrlFunctionsSuite] + enableSuite[GlutenFileMetadataStructRowIndexSuite] + // Row index metadata column support in Velox isn't ready yet, refer velox-9147 + .exclude("reading _tmp_metadata_row_index - not present in a table") + .exclude("reading _tmp_metadata_row_index - present in a table") + // Row index metadata column support in Velox isn't ready yet, refer velox-9147 + // enableSuite[GlutenParquetRowIndexSuite] override def getSQLQueryTestSettings: SQLQueryTestSettings = VeloxSQLQueryTestSettings } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala new file mode 100644 index 000000000000..d578b92c4c8a --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenDataFrameToSchemaSuite extends DataFrameToSchemaSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala new file mode 100644 index 000000000000..e3ba780530fd --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenDatasetUnpivotSuite extends DatasetUnpivotSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala new file mode 100644 index 000000000000..cc90f46e1a3d --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenLateralColumnAliasSuite extends LateralColumnAliasSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala new file mode 100644 index 000000000000..0887a7416fd8 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenParametersSuite extends ParametersSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala new file mode 100644 index 000000000000..2d1570be26c5 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenResolveDefaultColumnsSuite + extends ResolveDefaultColumnsSuite + with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala new file mode 100644 index 000000000000..323c5fbe1477 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenSubqueryHintPropagationSuite + extends SubqueryHintPropagationSuite + with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala new file mode 100644 index 000000000000..ae173ecd47f9 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenUrlFunctionsSuite extends UrlFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala new file mode 100644 index 000000000000..74893c5e51a2 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenDeltaBasedDeleteFromTableSuite + extends DeltaBasedDeleteFromTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala new file mode 100644 index 000000000000..25f377505c69 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenGroupBasedDeleteFromTableSuite + extends GroupBasedDeleteFromTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructRowIndexSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructRowIndexSuite.scala new file mode 100644 index 000000000000..aa62bc2ab18c --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructRowIndexSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenFileMetadataStructRowIndexSuite + extends FileMetadataStructRowIndexSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala new file mode 100644 index 000000000000..acf6a2b6384d --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait {} From 717b263d01310d94c0fa4ec506d148ef00367448 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 7 May 2024 09:25:38 +0800 Subject: [PATCH 005/402] [VL] CI: Gluten-it: Print planning time as well as execution time in test report (#5616) --- .../tpc/action/Parameterized.scala | 55 ++++++++------ .../integration/tpc/action/Queries.scala | 34 ++++++--- .../tpc/action/QueriesCompare.scala | 61 ++++++++++----- .../tpc/action/TableFormatter.scala | 74 +++++++++++++++++++ .../org/apache/spark/sql/QueryRunner.scala | 12 ++- 5 files changed, 183 insertions(+), 53 deletions(-) create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala index b4f7a53943e4..6fc4e66d6f05 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala @@ -25,7 +25,7 @@ import org.apache.gluten.integration.tpc.action.Actions.QuerySelector import scala.collection.immutable.Map import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer +import scala.collection.mutable.{ArrayBuffer, ListBuffer} class Parameterized( scale: Double, @@ -189,30 +189,15 @@ case class TestResultLine( succeed: Boolean, coordinate: Coordinate, rowCount: Option[Long], + planningTimeMillis: Option[Long], executionTimeMillis: Option[Long], metrics: Map[String, Long], errorMessage: Option[String]) -case class TestResultLines( - dimNames: Seq[String], - metricNames: Seq[String], - lines: Iterable[TestResultLine]) { - def print(): Unit = { - var fmt = "|%15s|%15s" - for (_ <- dimNames.indices) { - fmt = fmt + "|%20s" - } - for (_ <- metricNames.indices) { - fmt = fmt + "|%35s" - } - fmt = fmt + "|%30s|%30s|\n" - val fields = ArrayBuffer[String]("Query ID", "Succeed") - dimNames.foreach(dimName => fields.append(dimName)) - metricNames.foreach(metricName => fields.append(metricName)) - fields.append("Row Count") - fields.append("Query Time (Millis)") - printf(fmt, fields: _*) - lines.foreach { line => +object TestResultLine { + class Parser(dimNames: Seq[String], metricNames: Seq[String]) + extends TableFormatter.RowParser[TestResultLine] { + override def parse(line: TestResultLine): Seq[Any] = { val values = ArrayBuffer[Any](line.queryId, line.succeed) dimNames.foreach { dimName => val coordinate = line.coordinate.coordinate @@ -226,9 +211,32 @@ case class TestResultLines( values.append(metrics.getOrElse(metricName, "N/A")) } values.append(line.rowCount.getOrElse("N/A")) + values.append(line.planningTimeMillis.getOrElse("N/A")) values.append(line.executionTimeMillis.getOrElse("N/A")) - printf(fmt, values: _*) + values + } + } +} + +case class TestResultLines( + dimNames: Seq[String], + metricNames: Seq[String], + lines: Iterable[TestResultLine]) { + def print(): Unit = { + val fields = ListBuffer[String]("Query ID", "Succeed") + dimNames.foreach(dimName => fields.append(dimName)) + metricNames.foreach(metricName => fields.append(metricName)) + fields.append("Row Count") + fields.append("Planning Time (Millis)") + fields.append("Query Time (Millis)") + val formatter = TableFormatter.create[TestResultLine](fields: _*)( + new TestResultLine.Parser(dimNames, metricNames)) + + lines.foreach { line => + formatter.appendRow(line) } + + formatter.print(System.out) } } @@ -257,6 +265,7 @@ object Parameterized { succeed = true, coordinate, Some(resultRows.length), + Some(result.planningTimeMillis), Some(result.executionTimeMillis), result.metrics, None) @@ -266,7 +275,7 @@ object Parameterized { println( s"Error running query $id. " + s" Error: ${error.get}") - TestResultLine(id, succeed = false, coordinate, None, None, Map.empty, error) + TestResultLine(id, succeed = false, coordinate, None, None, None, Map.empty, error) } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala index c5f883189d29..edeb960fcba9 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala @@ -100,24 +100,36 @@ object Queries { queryId: String, testPassed: Boolean, rowCount: Option[Long], + planningTimeMillis: Option[Long], executionTimeMillis: Option[Long], errorMessage: Option[String]) + object TestResultLine { + implicit object Parser extends TableFormatter.RowParser[TestResultLine] { + override def parse(line: TestResultLine): Seq[Any] = { + Seq( + line.queryId, + line.testPassed, + line.rowCount.getOrElse("N/A"), + line.planningTimeMillis.getOrElse("N/A"), + line.executionTimeMillis.getOrElse("N/A")) + } + } + } + private def printResults(results: List[TestResultLine]): Unit = { - printf( - "|%15s|%15s|%30s|%30s|\n", + val formatter = TableFormatter.create[TestResultLine]( "Query ID", "Was Passed", "Row Count", + "Plan Time (Millis)", "Query Time (Millis)") + results.foreach { line => - printf( - "|%15s|%15s|%30s|%30s|\n", - line.queryId, - line.testPassed, - line.rowCount.getOrElse("N/A"), - line.executionTimeMillis.getOrElse("N/A")) + formatter.appendRow(line) } + + formatter.print(System.out) } private def aggregate(succeed: List[TestResultLine], name: String): List[TestResultLine] = { @@ -132,6 +144,9 @@ object Queries { if (r1.rowCount.nonEmpty && r2.rowCount.nonEmpty) Some(r1.rowCount.get + r2.rowCount.get) else None, + if (r1.planningTimeMillis.nonEmpty && r2.planningTimeMillis.nonEmpty) + Some(r1.planningTimeMillis.get + r2.planningTimeMillis.get) + else None, if (r1.executionTimeMillis.nonEmpty && r2.executionTimeMillis.nonEmpty) Some(r1.executionTimeMillis.get + r2.executionTimeMillis.get) else None, @@ -164,6 +179,7 @@ object Queries { id, testPassed = true, Some(resultRows.length), + Some(result.planningTimeMillis), Some(result.executionTimeMillis), None) } catch { @@ -172,7 +188,7 @@ object Queries { println( s"Error running query $id. " + s" Error: ${error.get}") - TestResultLine(id, testPassed = false, None, None, error) + TestResultLine(id, testPassed = false, None, None, None, error) } } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala index 5e8e2d6136f7..cfb3e7dc5378 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala @@ -101,37 +101,52 @@ object QueriesCompare { testPassed: Boolean, expectedRowCount: Option[Long], actualRowCount: Option[Long], + expectedPlanningTimeMillis: Option[Long], + actualPlanningTimeMillis: Option[Long], expectedExecutionTimeMillis: Option[Long], actualExecutionTimeMillis: Option[Long], errorMessage: Option[String]) + object TestResultLine { + implicit object Parser extends TableFormatter.RowParser[TestResultLine] { + override def parse(line: TestResultLine): Seq[Any] = { + val timeVariation = + if (line.expectedExecutionTimeMillis.nonEmpty && line.actualExecutionTimeMillis.nonEmpty) { + Some( + ((line.expectedExecutionTimeMillis.get - line.actualExecutionTimeMillis.get).toDouble + / line.actualExecutionTimeMillis.get.toDouble) * 100) + } else None + Seq( + line.queryId, + line.testPassed, + line.expectedRowCount.getOrElse("N/A"), + line.actualRowCount.getOrElse("N/A"), + line.expectedPlanningTimeMillis.getOrElse("N/A"), + line.actualPlanningTimeMillis.getOrElse("N/A"), + line.expectedExecutionTimeMillis.getOrElse("N/A"), + line.actualExecutionTimeMillis.getOrElse("N/A"), + timeVariation.map("%15.2f%%".format(_)).getOrElse("N/A")) + } + } + } + private def printResults(results: List[TestResultLine]): Unit = { - printf( - "|%15s|%15s|%30s|%30s|%30s|%30s|%30s|\n", + val formatter = TableFormatter.create[TestResultLine]( "Query ID", "Was Passed", "Expected Row Count", "Actual Row Count", + "Baseline Planning Time (Millis)", + "Planning Time (Millis)", "Baseline Query Time (Millis)", "Query Time (Millis)", "Query Time Variation") + results.foreach { line => - val timeVariation = - if (line.expectedExecutionTimeMillis.nonEmpty && line.actualExecutionTimeMillis.nonEmpty) { - Some( - ((line.expectedExecutionTimeMillis.get - line.actualExecutionTimeMillis.get).toDouble - / line.actualExecutionTimeMillis.get.toDouble) * 100) - } else None - printf( - "|%15s|%15s|%30s|%30s|%30s|%30s|%30s|\n", - line.queryId, - line.testPassed, - line.expectedRowCount.getOrElse("N/A"), - line.actualRowCount.getOrElse("N/A"), - line.expectedExecutionTimeMillis.getOrElse("N/A"), - line.actualExecutionTimeMillis.getOrElse("N/A"), - timeVariation.map("%15.2f%%".format(_)).getOrElse("N/A")) + formatter.appendRow(line) } + + formatter.print(System.out) } private def aggregate(succeed: List[TestResultLine], name: String): List[TestResultLine] = { @@ -149,6 +164,12 @@ object QueriesCompare { if (r1.actualRowCount.nonEmpty && r2.actualRowCount.nonEmpty) Some(r1.actualRowCount.get + r2.actualRowCount.get) else None, + if (r1.expectedPlanningTimeMillis.nonEmpty && r2.expectedPlanningTimeMillis.nonEmpty) + Some(r1.expectedPlanningTimeMillis.get + r2.expectedPlanningTimeMillis.get) + else None, + if (r1.actualPlanningTimeMillis.nonEmpty && r2.actualPlanningTimeMillis.nonEmpty) + Some(r1.actualPlanningTimeMillis.get + r2.actualPlanningTimeMillis.get) + else None, if (r1.expectedExecutionTimeMillis.nonEmpty && r2.expectedExecutionTimeMillis.nonEmpty) Some(r1.expectedExecutionTimeMillis.get + r2.expectedExecutionTimeMillis.get) else None, @@ -187,6 +208,8 @@ object QueriesCompare { testPassed = true, Some(expectedRows.length), Some(resultRows.length), + Some(expected.planningTimeMillis), + Some(result.planningTimeMillis), Some(expected.executionTimeMillis), Some(result.executionTimeMillis), None) @@ -198,6 +221,8 @@ object QueriesCompare { testPassed = false, Some(expectedRows.length), Some(resultRows.length), + Some(expected.planningTimeMillis), + Some(result.planningTimeMillis), Some(expected.executionTimeMillis), Some(result.executionTimeMillis), error) @@ -207,7 +232,7 @@ object QueriesCompare { println( s"Error running query $id. " + s" Error: ${error.get}") - TestResultLine(id, testPassed = false, None, None, None, None, error) + TestResultLine(id, testPassed = false, None, None, None, None, None, None, error) } } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala new file mode 100644 index 000000000000..cb6ab7ebd056 --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gluten.integration.tpc.action + +import java.io.{OutputStream, PrintStream} +import scala.collection.mutable + +trait TableFormatter[ROW <: Any] { + import TableFormatter._ + def appendRow(row: ROW): Unit + def print(s: OutputStream): Unit +} + +object TableFormatter { + def create[ROW <: Any](fields: String*)( + implicit parser: RowParser[ROW]): TableFormatter[ROW] = { + assert(fields.nonEmpty) + new Impl[ROW](Schema(fields), parser) + } + + private case class Schema(fields: Seq[String]) + + private class Impl[ROW <: Any](schema: Schema, parser: RowParser[ROW]) + extends TableFormatter[ROW] { + private val rows = mutable.ListBuffer[Seq[String]]() + + override def appendRow(row: ROW): Unit = { + val parsed = parser.parse(row) + assert(parsed.size == schema.fields.size) + rows += parsed.map(_.toString) + } + + override def print(s: OutputStream): Unit = { + val numFields = schema.fields.size + val widths = (0 until numFields) + .map { i => + rows.map(_(i).length).max max schema.fields(i).length + } + .map(_ + 1) + val pBuilder = StringBuilder.newBuilder + pBuilder ++= "|" + widths.foreach { w => + pBuilder ++= s"%${w}s|" + } + val pattern = pBuilder.toString() + val printer = new PrintStream(s) + printer.println(String.format(pattern, schema.fields: _*)) + rows.foreach { r => + printer.println(String.format(pattern, r: _*)) + } + printer.flush() + printer.close() + } + } + + trait RowParser[ROW <: Any] { + def parse(row: ROW): Seq[Any] + } +} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala index 332e56043c45..a5b699a1ae48 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala @@ -89,9 +89,11 @@ object QueryRunner { if (explain) { df.explain(extended = true) } - val millis = (System.nanoTime() - prev) / 1000000L + val planMillis = + df.queryExecution.tracker.phases.values.map(p => p.endTimeMs - p.startTimeMs).sum + val totalMillis = (System.nanoTime() - prev) / 1000000L val collectedMetrics = metrics.map(name => (name, em.getMetricValue(name))).toMap - RunResult(rows, millis, collectedMetrics) + RunResult(rows, planMillis, totalMillis - planMillis, collectedMetrics) } finally { sc.removeSparkListener(metricsListener) killTaskListener.foreach(l => { @@ -124,7 +126,11 @@ object QueryRunner { } -case class RunResult(rows: Seq[Row], executionTimeMillis: Long, metrics: Map[String, Long]) +case class RunResult( + rows: Seq[Row], + planningTimeMillis: Long, + executionTimeMillis: Long, + metrics: Map[String, Long]) class MetricsListener(em: ExecutorMetrics) extends SparkListener { override def onExecutorMetricsUpdate( From fea47053c9716ad1cc2433403312b348889ca959 Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Tue, 7 May 2024 09:43:12 +0800 Subject: [PATCH 006/402] [GLUTEN-5618][CH] Fix 'Position x is out of bound in Block' error when executing count distinct (#5619) When excuting count distinct, the group by keys are also in the count distinct expression, it will throw 'Position x is out of bound in Block' error or core dump. RC: CH backend will remove the duplicated column when executing pipeline. Close #5618. --- .../clickhouse/CHSparkPlanExecApi.scala | 7 +- .../GlutenClickhouseCountDistinctSuite.scala | 98 +++++++++++++++++++ .../CountDistinctWithoutExpand.scala | 6 +- 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index ee8b7dd4540d..64090af287e4 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -194,12 +194,13 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { child: SparkPlan): HashAggregateExecBaseTransformer = CHHashAggregateExecTransformer( requiredChildDistributionExpressions, - groupingExpressions, + groupingExpressions.distinct, aggregateExpressions, aggregateAttributes, initialInputBufferOffset, - resultExpressions, - child) + resultExpressions.distinct, + child + ) /** Generate HashAggregateExecPullOutHelper */ override def genHashAggregateExecPullOutHelper( diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala index b12f886e5d6d..1b954df22eac 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala @@ -115,4 +115,102 @@ class GlutenClickhouseCountDistinctSuite extends GlutenClickHouseWholeStageTrans "values (0, null,1), (0,null,1), (1, 1,1), (2, 2, 1) ,(2,2,2),(3,3,3) as data(a,b,c)" compareResultsAgainstVanillaSpark(sql, true, { _ => }) } + + test( + "Gluten-5618: [CH] Fix 'Position x is out of bound in Block' error " + + "when executing count distinct") { + + withSQLConf(("spark.gluten.sql.countDistinctWithoutExpand", "false")) { + val sql = + """ + |select count(distinct a, b, c) from + |values (0, null, 1), (1, 1, 1), (2, 2, 1), (1, 2, 1) ,(2, 2, 2) as data(a,b,c) group by c + |""".stripMargin + + compareResultsAgainstVanillaSpark( + sql, + true, + { + df => + { + + val planExecs = df.queryExecution.executedPlan.collect { + case aggTransformer: HashAggregateExecBaseTransformer => aggTransformer + } + + planExecs.head.aggregateExpressions.foreach { + expr => assert(expr.toString().startsWith("count(")) + } + planExecs(1).aggregateExpressions.foreach { + expr => assert(expr.toString().startsWith("partial_count(")) + } + } + } + ) + } + + val sql = + """ + |select count(distinct a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) + |from values + |(0, null, 1, 0, null, 1, 0, 5, 1, 0), + |(null, 1, 1, null, 1, 1, null, 1, 1, 3), + |(2, 2, 1, 2, 2, 1, 2, 2, 1, 2), + |(1, 2, null, 1, 2, null, 1, 2, 3, 1), + |(2, 2, 2, 2, 2, 2, 2, 2, 2, 2) + |as data(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) + |group by a10 + |""".stripMargin + + compareResultsAgainstVanillaSpark( + sql, + true, + { + df => + { + + val planExecs = df.queryExecution.executedPlan.collect { + case aggTransformer: HashAggregateExecBaseTransformer => aggTransformer + } + + planExecs.head.aggregateExpressions.foreach { + expr => assert(expr.toString().startsWith("count(")) + } + planExecs(1).aggregateExpressions.foreach { + expr => assert(expr.toString().startsWith("partial_count(")) + } + } + } + ) + + val sql1 = + """ + |select count(distinct a, b, c) + |from + |values (0, null, 1), (1, 1, 1), (null, 2, 1), (1, 2, 1) ,(2, 2, null) + |as data(a,b,c) + |group by c + |""".stripMargin + + compareResultsAgainstVanillaSpark( + sql1, + true, + { + df => + { + + val planExecs = df.queryExecution.executedPlan.collect { + case aggTransformer: HashAggregateExecBaseTransformer => aggTransformer + } + + planExecs.head.aggregateExpressions.foreach { + expr => assert(expr.toString().startsWith("countdistinct(")) + } + planExecs(1).aggregateExpressions.foreach { + expr => assert(expr.toString().startsWith("partial_countdistinct(")) + } + } + } + ) + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala index 43cc68eadbe5..82051baeebc7 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala @@ -36,7 +36,11 @@ object CountDistinctWithoutExpand extends Rule[LogicalPlan] { GlutenConfig.getConf.enableGluten && GlutenConfig.getConf.enableCountDistinctWithoutExpand ) { plan.transformAllExpressionsWithPruning(_.containsPattern(AGGREGATE_EXPRESSION)) { - case ae: AggregateExpression if ae.isDistinct && ae.aggregateFunction.isInstanceOf[Count] => + case ae: AggregateExpression + if ae.isDistinct && ae.aggregateFunction.isInstanceOf[Count] && + // The maximum number of arguments for aggregate function with Nullable types in CH + // backend is 8 + ae.aggregateFunction.children.size <= 8 => ae.copy( aggregateFunction = CountDistinct.apply(ae.aggregateFunction.asInstanceOf[Count].children), From 0a40b2433f8314115fd3c713cbb044e684b57b6b Mon Sep 17 00:00:00 2001 From: Zouxxyy Date: Tue, 7 May 2024 10:26:56 +0800 Subject: [PATCH 007/402] [CORE] Only return columns of partitions that require read for iceberg (#5624) --- .../execution/IcebergScanTransformer.scala | 4 +-- .../source/GlutenIcebergSourceUtil.scala | 30 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala index 9bb33678a9df..6e079bf7e10a 100644 --- a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala +++ b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala @@ -48,7 +48,7 @@ case class IcebergScanTransformer( override def filterExprs(): Seq[Expression] = pushdownFilters.getOrElse(Seq.empty) - override def getPartitionSchema: StructType = GlutenIcebergSourceUtil.getPartitionSchema(scan) + override def getPartitionSchema: StructType = GlutenIcebergSourceUtil.getReadPartitionSchema(scan) override def getDataSchema: StructType = new StructType() @@ -63,7 +63,7 @@ case class IcebergScanTransformer( filteredPartitions, outputPartitioning) groupedPartitions.zipWithIndex.map { - case (p, index) => GlutenIcebergSourceUtil.genSplitInfo(p, index) + case (p, index) => GlutenIcebergSourceUtil.genSplitInfo(p, index, getPartitionSchema) } } diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala index 2b4f54aef141..6b67e763648b 100644 --- a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala +++ b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala @@ -34,7 +34,10 @@ import scala.collection.JavaConverters._ object GlutenIcebergSourceUtil { - def genSplitInfo(inputPartition: InputPartition, index: Int): SplitInfo = inputPartition match { + def genSplitInfo( + inputPartition: InputPartition, + index: Int, + readPartitionSchema: StructType): SplitInfo = inputPartition match { case partition: SparkInputPartition => val paths = new JArrayList[String]() val starts = new JArrayList[JLong]() @@ -50,8 +53,8 @@ object GlutenIcebergSourceUtil { paths.add(filePath) starts.add(task.start()) lengths.add(task.length()) - partitionColumns.add(getPartitionColumns(task)) - deleteFilesList.add(task.deletes()); + partitionColumns.add(getPartitionColumns(task, readPartitionSchema)) + deleteFilesList.add(task.deletes()) val currentFileFormat = convertFileFormat(task.file().format()) if (fileFormat == ReadFileFormat.UnknownFormat) { fileFormat = currentFileFormat @@ -94,7 +97,7 @@ object GlutenIcebergSourceUtil { throw new GlutenNotSupportException("Only support iceberg SparkBatchQueryScan.") } - def getPartitionSchema(sparkScan: Scan): StructType = sparkScan match { + def getReadPartitionSchema(sparkScan: Scan): StructType = sparkScan match { case scan: SparkBatchQueryScan => val tasks = scan.tasks().asScala asFileScanTask(tasks.toList).foreach { @@ -102,7 +105,16 @@ object GlutenIcebergSourceUtil { val spec = task.spec() if (spec.isPartitioned) { var partitionSchema = new StructType() - val partitionFields = spec.partitionType().fields().asScala + val readFields = scan.readSchema().fields.map(_.name).toSet + // Iceberg will generate some non-table fields as partition fields, such as x_bucket, + // which will not appear in readFields, they also cannot be filtered. + val tableFields = spec.schema().columns().asScala.map(_.name()).toSet + val partitionFields = + spec + .partitionType() + .fields() + .asScala + .filter(f => !tableFields.contains(f.name) || readFields.contains(f.name())) partitionFields.foreach { field => TypeUtil.validatePartitionColumnType(field.`type`().typeId()) @@ -130,12 +142,16 @@ object GlutenIcebergSourceUtil { } } - private def getPartitionColumns(task: FileScanTask): JHashMap[String, String] = { + private def getPartitionColumns( + task: FileScanTask, + readPartitionSchema: StructType): JHashMap[String, String] = { val partitionColumns = new JHashMap[String, String]() + val readPartitionFields = readPartitionSchema.fields.map(_.name).toSet val spec = task.spec() val partition = task.partition() if (spec.isPartitioned) { - val partitionFields = spec.partitionType().fields().asScala + val partitionFields = + spec.partitionType().fields().asScala.filter(f => readPartitionFields.contains(f.name())) partitionFields.zipWithIndex.foreach { case (field, index) => val partitionValue = partition.get(index, field.`type`().typeId().javaClass()) From d18fd7ad502d0bf406f091ea4912a259bf32fac9 Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Tue, 7 May 2024 09:48:35 +0530 Subject: [PATCH 008/402] [VL] Enable unix_seconds Spark function (#5602) [VL] Enable unix_seconds Spark function. --- .../apache/gluten/utils/CHExpressionUtil.scala | 1 + .../execution/ScalarFunctionsValidateSuite.scala | 15 +++++++++++++++ docs/velox-backend-support-progress.md | 9 +++++---- .../gluten/expression/ExpressionMappings.scala | 1 + .../gluten/expression/ExpressionNames.scala | 1 + 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index bbe65034a0a3..0e645d039840 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -178,6 +178,7 @@ object CHExpressionUtil { ARRAYS_ZIP -> DefaultValidator(), DATE_FROM_UNIX_DATE -> DefaultValidator(), UNIX_DATE -> DefaultValidator(), + UNIX_SECONDS -> DefaultValidator(), MONOTONICALLY_INCREASING_ID -> DefaultValidator(), SPARK_PARTITION_ID -> DefaultValidator(), URL_DECODE -> DefaultValidator(), diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 6fb45141188f..0a7a7d6cb9a6 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -844,6 +844,21 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("unix_seconds") { + withTempPath { + path => + val t1 = Timestamp.valueOf("2024-08-22 10:10:10.010") + val t2 = Timestamp.valueOf("2014-12-31 00:00:00.012") + val t3 = Timestamp.valueOf("1968-12-31 23:59:59.001") + Seq(t1, t2, t3).toDF("t").write.parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("view") + runQueryAndCompare("select unix_seconds(t) from view") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("unix_millis") { withTempPath { path => diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 90f4fda9ea4b..e98587efbbe4 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -338,8 +338,8 @@ Gluten supports 199 functions. (Drag to right to see all data types) | second | second | | S | | | | | | | | | | | | | | | | | | | | | session_window | | | | | | | | | | | | | | | | | | | | | | | | timestamp | | | | | | | | | | | | | | | | | | | | | | | -| timestamp_micros | | timestamp_micros | S | | | | | | | | | | | | | | | | | | | | -| timestamp_millis | | timestamp_millis | S | | | | | | | | | | | | | | | | | | | | +| timestamp_micros | | timestamp_micros | S | | | | | | | | | | | | | | | | | | | | +| timestamp_millis | | timestamp_millis | S | | | | | | | | | | | | | | | | | | | | | timestamp_seconds | | | | | | | | | | | | | | | | | | | | | | | | to_date | | | S | | | | | | | | | S | S | | | | | | | | | | | to_timestamp | | | | | | | | | | | | | | | | | | | | | | | @@ -347,8 +347,9 @@ Gluten supports 199 functions. (Drag to right to see all data types) | to_utc_timestamp | | | | | | | | | | | | | | | | | | | | | | | | trunc | | | | | | | | | | | | | | | | | | | | | | | | unix_timestamp | | unix_timestamp | | | | | | | | | | | | | | | | | | | | | -| unix_millis | | unix_millis | S | | | | | | | | | | | | | | | | | | | | -| unix_micros | | unix_micros | S | | | | | | | | | | | | | | | | | | | | +| unix_seconds | | unix_seconds | S | | | | | | | | | | | | | | | | | | | | +| unix_millis | | unix_millis | S | | | | | | | | | | | | | | | | | | | | +| unix_micros | | unix_micros | S | | | | | | | | | | | | | | | | | | | | | weekday | | | S | | | | | | | | | S | | | | | | | | | | | | weekofyear | week,week_of_year | | S | | | | | | | | | | | | | | | | | | | | | window | | | | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 19a77e515eec..920b6fab823e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -189,6 +189,7 @@ object ExpressionMappings { Sig[MakeYMInterval](MAKE_YM_INTERVAL), Sig[ToUTCTimestamp](TO_UTC_TIMESTAMP), Sig[FromUTCTimestamp](FROM_UTC_TIMESTAMP), + Sig[UnixSeconds](UNIX_SECONDS), Sig[UnixMillis](UNIX_MILLIS), Sig[UnixMicros](UNIX_MICROS), Sig[MillisToTimestamp](TIMESTAMP_MILLIS), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 1e0d86a66c4f..2ad1c6f739c5 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -209,6 +209,7 @@ object ExpressionNames { final val MAKE_YM_INTERVAL = "make_ym_interval" final val TO_UTC_TIMESTAMP = "to_utc_timestamp" final val FROM_UTC_TIMESTAMP = "from_utc_timestamp" + final val UNIX_SECONDS = "unix_seconds" final val UNIX_MILLIS = "unix_millis" final val UNIX_MICROS = "unix_micros" final val TIMESTAMP_MILLIS = "timestamp_millis" From ca7d3f2fe2d8be7fd03a68f9232b7c7cf97ea811 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Tue, 7 May 2024 12:20:16 +0800 Subject: [PATCH 009/402] [VL] Daily Update Velox Version (2024_05_07) (#5628) auto rebase successed --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index c26bedd5e9af..987934204ac5 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_06 +VELOX_BRANCH=2024_05_07 VELOX_HOME="" #Set on run gluten on HDFS From 2127adcc51c67d4f499434b8dc71ca4c744021e4 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 7 May 2024 13:45:17 +0800 Subject: [PATCH 010/402] [GLUTEN-5611] [VL] Avoid trigger Spark memory listener when native memory request can be handled internally (#5631) --- cpp/velox/memory/VeloxMemoryManager.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index c8d13b871062..93eb93f6bed9 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -40,6 +40,9 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { } uint64_t growCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { + if (targetBytes == 0) { + return 0; + } std::lock_guard l(mutex_); return growPoolLocked(pool, targetBytes); } @@ -66,8 +69,8 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { uint64_t shrinkCapacity( const std::vector>& pools, uint64_t targetBytes, - bool allowSpill = true, - bool allowAbort = false) override { + bool allowSpill, + bool allowAbort) override { facebook::velox::exec::MemoryReclaimer::Stats status; GLUTEN_CHECK(pools.size() == 1, "Should shrink a single pool at a time"); std::lock_guard l(mutex_); // FIXME: Do we have recursive locking for this mutex? @@ -102,13 +105,14 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { reserved, "Unexpected: Failed to reserve " + std::to_string(bytes) + " bytes although there is enough space, free bytes: " + std::to_string(freeBytes)); + return 0; } listener_->allocationChanged(bytes); return pool->grow(bytes, bytes); } uint64_t releaseMemoryLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { - uint64_t freeBytes = pool->shrink(0); + uint64_t freeBytes = pool->shrink(bytes); listener_->allocationChanged(-freeBytes); return freeBytes; } From 18af4bc3ce4c3e685ad63c869880f8b63d48dc1c Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 7 May 2024 14:49:03 +0800 Subject: [PATCH 011/402] [VL] RAS: Include rewrite rules used by RewriteSparkPlanRulesManager in EnumeratedTransform (#5575) --- .../clickhouse/CHSparkPlanExecApi.scala | 3 +- .../CHHashAggregateExecTransformer.scala | 6 +- .../velox/VeloxSparkPlanExecApi.scala | 3 +- .../HashAggregateExecTransformer.scala | 14 +-- .../gluten/backendsapi/SparkPlanExecApi.scala | 1 - .../HashAggregateExecBaseTransformer.scala | 30 +++++- .../apache/gluten/extension/RewriteIn.scala | 7 +- .../columnar/MiscColumnarRules.scala | 18 ++-- ...ngleNode.scala => OffloadSingleNode.scala} | 95 +++++++------------ .../columnar/TransformHintRule.scala | 33 +------ .../columnar/enumerated/ConditionedRule.scala | 51 ---------- .../enumerated/EnumeratedApplier.scala | 5 +- .../enumerated/EnumeratedTransform.scala | 56 ++++------- .../enumerated/PushFilterToScan.scala | 27 +++--- .../columnar/enumerated/RasOffload.scala | 84 ++++++++++++++++ ...regate.scala => RasOffloadAggregate.scala} | 35 ++----- ...entFilter.scala => RasOffloadFilter.scala} | 16 +--- ...terRemoveRule.scala => RemoveFilter.scala} | 2 +- .../columnar/heuristic/HeuristicApplier.scala | 1 + .../{ => rewrite}/PullOutPostProject.scala | 14 ++- .../{ => rewrite}/PullOutPreProject.scala | 9 +- .../rewrite}/RewriteCollect.scala | 9 +- .../RewriteMultiChildrenCount.scala | 7 +- .../columnar/rewrite/RewriteSingleNode.scala | 48 ++++++++++ .../RewriteSparkPlanRulesManager.scala | 17 +--- .../RewriteTypedImperativeAggregate.scala | 7 +- .../gluten/planner/cost/GlutenCostModel.scala | 6 +- .../GlutenFormatWriterInjectsBase.scala | 3 +- 28 files changed, 295 insertions(+), 312 deletions(-) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{TransformSingleNode.scala => OffloadSingleNode.scala} (88%) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ConditionedRule.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/{ImplementAggregate.scala => RasOffloadAggregate.scala} (50%) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/{ImplementFilter.scala => RasOffloadFilter.scala} (75%) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/{FilterRemoveRule.scala => RemoveFilter.scala} (97%) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{ => rewrite}/PullOutPostProject.scala (92%) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{ => rewrite}/PullOutPreProject.scala (96%) rename gluten-core/src/main/scala/org/apache/gluten/extension/{ => columnar/rewrite}/RewriteCollect.scala (93%) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{ => rewrite}/RewriteMultiChildrenCount.scala (93%) create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{ => rewrite}/RewriteSparkPlanRulesManager.scala (91%) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{ => rewrite}/RewriteTypedImperativeAggregate.scala (91%) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 64090af287e4..a9a12a3ea2ce 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -204,10 +204,9 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { /** Generate HashAggregateExecPullOutHelper */ override def genHashAggregateExecPullOutHelper( - groupingExpressions: Seq[NamedExpression], aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute]): HashAggregateExecPullOutBaseHelper = - CHHashAggregateExecPullOutHelper(groupingExpressions, aggregateExpressions, aggregateAttributes) + CHHashAggregateExecPullOutHelper(aggregateExpressions, aggregateAttributes) /** * If there are expressions (not field reference) in the partitioning's children, add a projection diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala index 82c492f4c8cf..d4f2f9eb3874 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala @@ -411,13 +411,9 @@ case class CHHashAggregateExecTransformer( } case class CHHashAggregateExecPullOutHelper( - groupingExpressions: Seq[NamedExpression], aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute]) - extends HashAggregateExecPullOutBaseHelper( - groupingExpressions, - aggregateExpressions, - aggregateAttributes) { + extends HashAggregateExecPullOutBaseHelper { /** This method calculates the output attributes of Aggregation. */ override protected def getAttrForAggregateExprs: List[Attribute] = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index a55aa1817ffd..0a9f3ef65fd1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -330,10 +330,9 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { /** Generate HashAggregateExecPullOutHelper */ override def genHashAggregateExecPullOutHelper( - groupingExpressions: Seq[NamedExpression], aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute]): HashAggregateExecPullOutBaseHelper = - HashAggregateExecPullOutHelper(groupingExpressions, aggregateExpressions, aggregateAttributes) + HashAggregateExecPullOutHelper(aggregateExpressions, aggregateAttributes) override def genColumnarShuffleExchange( shuffle: ShuffleExchangeExec, diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala index 0a9904206b42..f0a7ea1801d1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.extension.columnar.RewriteTypedImperativeAggregate +import org.apache.gluten.extension.columnar.rewrite.RewriteTypedImperativeAggregate import org.apache.gluten.substrait.`type`.{TypeBuilder, TypeNode} import org.apache.gluten.substrait.{AggregationParams, SubstraitContext} import org.apache.gluten.substrait.expression.{AggregateFunctionNode, ExpressionBuilder, ExpressionNode, ScalarFunctionNode} @@ -60,6 +60,12 @@ abstract class HashAggregateExecTransformer( resultExpressions, child) { + override def output: Seq[Attribute] = { + // TODO: We should have a check to make sure the returned schema actually matches the output + // data. Since "resultExpressions" is not actually in used by Velox. + super.output + } + override def doTransform(context: SubstraitContext): TransformContext = { val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) @@ -793,13 +799,9 @@ case class FlushableHashAggregateExecTransformer( } case class HashAggregateExecPullOutHelper( - groupingExpressions: Seq[NamedExpression], aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute]) - extends HashAggregateExecPullOutBaseHelper( - groupingExpressions, - aggregateExpressions, - aggregateAttributes) { + extends HashAggregateExecPullOutBaseHelper { /** This method calculates the output attributes of Aggregation. */ override protected def getAttrForAggregateExprs: List[Attribute] = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index cfa1a4e53cec..f5e08a05d7a1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -102,7 +102,6 @@ trait SparkPlanExecApi { /** Generate HashAggregateExecPullOutHelper */ def genHashAggregateExecPullOutHelper( - groupingExpressions: Seq[NamedExpression], aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute]): HashAggregateExecPullOutBaseHelper diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala index baf88c727b01..49a9ee1e816a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala @@ -172,12 +172,32 @@ abstract class HashAggregateExecBaseTransformer( validation: Boolean = false): RelNode } -abstract class HashAggregateExecPullOutBaseHelper( - groupingExpressions: Seq[NamedExpression], - aggregateExpressions: Seq[AggregateExpression], - aggregateAttributes: Seq[Attribute]) { +object HashAggregateExecBaseTransformer { + + private def getInitialInputBufferOffset(agg: BaseAggregateExec): Int = agg match { + case a: HashAggregateExec => a.initialInputBufferOffset + case a: ObjectHashAggregateExec => a.initialInputBufferOffset + case a: SortAggregateExec => a.initialInputBufferOffset + } + + def from(agg: BaseAggregateExec)( + childConverter: SparkPlan => SparkPlan = p => p): HashAggregateExecBaseTransformer = { + BackendsApiManager.getSparkPlanExecApiInstance + .genHashAggregateExecTransformer( + agg.requiredChildDistributionExpressions, + agg.groupingExpressions, + agg.aggregateExpressions, + agg.aggregateAttributes, + getInitialInputBufferOffset(agg), + agg.resultExpressions, + childConverter(agg.child) + ) + } +} + +trait HashAggregateExecPullOutBaseHelper { // The direct outputs of Aggregation. - lazy val allAggregateResultAttributes: List[Attribute] = + def allAggregateResultAttributes(groupingExpressions: Seq[NamedExpression]): List[Attribute] = groupingExpressions.map(ConverterUtils.getAttrFromExpr(_)).toList ::: getAttrForAggregateExprs diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala index b508f3eff8f6..565b9bb19306 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala @@ -16,8 +16,9 @@ */ package org.apache.gluten.extension +import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode + import org.apache.spark.sql.catalyst.expressions.{EqualTo, Expression, In, Or} -import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, SparkPlan} import org.apache.spark.sql.types.StructType @@ -32,7 +33,7 @@ import org.apache.spark.sql.types.StructType * * TODO: Remove this rule once Velox support the list option in `In` is not literal. */ -object RewriteIn extends Rule[SparkPlan] { +object RewriteIn extends RewriteSingleNode { private def shouldRewrite(e: Expression): Boolean = { e match { @@ -58,7 +59,7 @@ object RewriteIn extends Rule[SparkPlan] { } } - override def apply(plan: SparkPlan): SparkPlan = { + override def rewrite(plan: SparkPlan): SparkPlan = { plan match { // TODO: Support datasource v2 case scan: FileSourceScanExec if scan.dataFilters.exists(_.find(shouldRewrite).isDefined) => diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala index 02a466b6ae2a..068f62e498ce 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala @@ -30,12 +30,12 @@ object MiscColumnarRules { object TransformPreOverrides { def apply(): TransformPreOverrides = { TransformPreOverrides( - List(TransformFilter()), + List(OffloadFilter()), List( - TransformOthers(), - TransformAggregate(), - TransformExchange(), - TransformJoin() + OffloadOthers(), + OffloadAggregate(), + OffloadExchange(), + OffloadJoin() ) ) } @@ -43,17 +43,17 @@ object MiscColumnarRules { // This rule will conduct the conversion from Spark plan to the plan transformer. case class TransformPreOverrides( - topDownRules: Seq[TransformSingleNode], - bottomUpRules: Seq[TransformSingleNode]) + topDownRules: Seq[OffloadSingleNode], + bottomUpRules: Seq[OffloadSingleNode]) extends Rule[SparkPlan] with LogLevelUtil { @transient private val planChangeLogger = new PlanChangeLogger[SparkPlan]() def apply(plan: SparkPlan): SparkPlan = { val plan0 = - topDownRules.foldLeft(plan)((p, rule) => p.transformDown { case p => rule.impl(p) }) + topDownRules.foldLeft(plan)((p, rule) => p.transformDown { case p => rule.offload(p) }) val plan1 = - bottomUpRules.foldLeft(plan0)((p, rule) => p.transformUp { case p => rule.impl(p) }) + bottomUpRules.foldLeft(plan0)((p, rule) => p.transformUp { case p => rule.offload(p) }) planChangeLogger.logRule(ruleName, plan, plan1) plan1 } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala similarity index 88% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformSingleNode.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 760929bbd806..84a2ec5c6ec8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -40,13 +40,20 @@ import org.apache.spark.sql.execution.python.{ArrowEvalPythonExec, BatchEvalPyth import org.apache.spark.sql.execution.window.{WindowExec, WindowGroupLimitExecShim} import org.apache.spark.sql.hive.HiveTableScanExecTransformer -sealed trait TransformSingleNode extends Logging { - def impl(plan: SparkPlan): SparkPlan +/** + * Converts a vanilla Spark plan node into Gluten plan node. Gluten plan is supposed to be executed + * in native, and the internals of execution is subject by backend's implementation. + * + * Note: Only the current plan node is supposed to be open to modification. Do not access or modify + * the children node. Tree-walking is done by caller of this trait. + */ +sealed trait OffloadSingleNode extends Logging { + def offload(plan: SparkPlan): SparkPlan } // Aggregation transformation. -case class TransformAggregate() extends TransformSingleNode with LogLevelUtil { - override def impl(plan: SparkPlan): SparkPlan = plan match { +case class OffloadAggregate() extends OffloadSingleNode with LogLevelUtil { + override def offload(plan: SparkPlan): SparkPlan = plan match { case plan if TransformHints.isNotTransformable(plan) => plan case agg: HashAggregateExec => @@ -69,19 +76,6 @@ case class TransformAggregate() extends TransformSingleNode with LogLevelUtil { val aggChild = plan.child - def transformHashAggregate(): GlutenPlan = { - BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - plan.requiredChildDistributionExpressions, - plan.groupingExpressions, - plan.aggregateExpressions, - plan.aggregateAttributes, - plan.initialInputBufferOffset, - plan.resultExpressions, - aggChild - ) - } - // If child's output is empty, fallback or offload both the child and aggregation. if ( aggChild.output.isEmpty && BackendsApiManager.getSettings @@ -91,9 +85,9 @@ case class TransformAggregate() extends TransformSingleNode with LogLevelUtil { case _: TransformSupport => // If the child is transformable, transform aggregation as well. logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - transformHashAggregate() + HashAggregateExecBaseTransformer.from(plan)() case p: SparkPlan if PlanUtil.isGlutenTableCache(p) => - transformHashAggregate() + HashAggregateExecBaseTransformer.from(plan)() case _ => // If the child is not transformable, do not transform the agg. TransformHints.tagNotTransformable(plan, "child output schema is empty") @@ -101,14 +95,14 @@ case class TransformAggregate() extends TransformSingleNode with LogLevelUtil { } } else { logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - transformHashAggregate() + HashAggregateExecBaseTransformer.from(plan)() } } } // Exchange transformation. -case class TransformExchange() extends TransformSingleNode with LogLevelUtil { - override def impl(plan: SparkPlan): SparkPlan = plan match { +case class OffloadExchange() extends OffloadSingleNode with LogLevelUtil { + override def offload(plan: SparkPlan): SparkPlan = plan match { case plan if TransformHints.isNotTransformable(plan) => plan case plan: ShuffleExchangeExec => @@ -131,10 +125,10 @@ case class TransformExchange() extends TransformSingleNode with LogLevelUtil { } // Join transformation. -case class TransformJoin() extends TransformSingleNode with LogLevelUtil { - import TransformJoin._ +case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { + import OffloadJoin._ - override def impl(plan: SparkPlan): SparkPlan = { + override def offload(plan: SparkPlan): SparkPlan = { if (TransformHints.isNotTransformable(plan)) { logDebug(s"Columnar Processing for ${plan.getClass} is under row guard.") plan match { @@ -223,7 +217,7 @@ case class TransformJoin() extends TransformSingleNode with LogLevelUtil { } -object TransformJoin { +object OffloadJoin { private def getSparkSupportedBuildSide(plan: ShuffledHashJoinExec): BuildSide = { plan.joinType match { case LeftOuter | LeftSemi => BuildRight @@ -238,11 +232,11 @@ object TransformJoin { } // Filter transformation. -case class TransformFilter() extends TransformSingleNode with LogLevelUtil { - import TransformOthers._ +case class OffloadFilter() extends OffloadSingleNode with LogLevelUtil { + import OffloadOthers._ private val replace = new ReplaceSingleNode() - override def impl(plan: SparkPlan): SparkPlan = plan match { + override def offload(plan: SparkPlan): SparkPlan = plan match { case filter: FilterExec => genFilterExec(filter) case other => other @@ -286,14 +280,14 @@ case class TransformFilter() extends TransformSingleNode with LogLevelUtil { } // Other transformations. -case class TransformOthers() extends TransformSingleNode with LogLevelUtil { - import TransformOthers._ +case class OffloadOthers() extends OffloadSingleNode with LogLevelUtil { + import OffloadOthers._ private val replace = new ReplaceSingleNode() - override def impl(plan: SparkPlan): SparkPlan = replace.doReplace(plan) + override def offload(plan: SparkPlan): SparkPlan = replace.doReplace(plan) } -object TransformOthers { +object OffloadOthers { // Utility to replace single node within transformed Gluten node. // Children will be preserved as they are as children of the output node. // @@ -333,35 +327,16 @@ object TransformOthers { ProjectExecTransformer(plan.projectList, columnarChild) case plan: SortAggregateExec => logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - plan.requiredChildDistributionExpressions, - plan.groupingExpressions, - plan.aggregateExpressions, - plan.aggregateAttributes, - plan.initialInputBufferOffset, - plan.resultExpressions, - plan.child match { - case sort: SortExecTransformer if !sort.global => - sort.child - case sort: SortExec if !sort.global => - sort.child - case _ => plan.child - } - ) + HashAggregateExecBaseTransformer.from(plan) { + case sort: SortExecTransformer if !sort.global => + sort.child + case sort: SortExec if !sort.global => + sort.child + case other => other + } case plan: ObjectHashAggregateExec => - val child = plan.child logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - plan.requiredChildDistributionExpressions, - plan.groupingExpressions, - plan.aggregateExpressions, - plan.aggregateAttributes, - plan.initialInputBufferOffset, - plan.resultExpressions, - child - ) + HashAggregateExecBaseTransformer.from(plan)() case plan: UnionExec => val children = plan.children logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index ea934425f144..3c3d23ccc5cc 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -355,40 +355,13 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { .genFilterExecTransformer(plan.condition, plan.child) transformer.doValidate().tagOnFallback(plan) case plan: HashAggregateExec => - val transformer = BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - plan.requiredChildDistributionExpressions, - plan.groupingExpressions, - plan.aggregateExpressions, - plan.aggregateAttributes, - plan.initialInputBufferOffset, - plan.resultExpressions, - plan.child - ) + val transformer = HashAggregateExecBaseTransformer.from(plan)() transformer.doValidate().tagOnFallback(plan) case plan: SortAggregateExec => - val transformer = BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - plan.requiredChildDistributionExpressions, - plan.groupingExpressions, - plan.aggregateExpressions, - plan.aggregateAttributes, - plan.initialInputBufferOffset, - plan.resultExpressions, - plan.child - ) + val transformer = HashAggregateExecBaseTransformer.from(plan)() transformer.doValidate().tagOnFallback(plan) case plan: ObjectHashAggregateExec => - val transformer = BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - plan.requiredChildDistributionExpressions, - plan.groupingExpressions, - plan.aggregateExpressions, - plan.aggregateAttributes, - plan.initialInputBufferOffset, - plan.resultExpressions, - plan.child - ) + val transformer = HashAggregateExecBaseTransformer.from(plan)() transformer.doValidate().tagOnFallback(plan) case plan: UnionExec => val transformer = ColumnarUnionExec(plan.children) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ConditionedRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ConditionedRule.scala deleted file mode 100644 index 33d99f5f7bea..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ConditionedRule.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension.columnar.enumerated - -import org.apache.gluten.extension.columnar.validator.Validator -import org.apache.gluten.ras.rule.{RasRule, Shape} - -import org.apache.spark.sql.execution.SparkPlan - -object ConditionedRule { - trait PreCondition { - def apply(node: SparkPlan): Boolean - } - - object PreCondition { - implicit class FromValidator(validator: Validator) extends PreCondition { - override def apply(node: SparkPlan): Boolean = { - validator.validate(node) match { - case Validator.Passed => true - case Validator.Failed(reason) => false - } - } - } - } - - def wrap(rule: RasRule[SparkPlan], cond: ConditionedRule.PreCondition): RasRule[SparkPlan] = { - new RasRule[SparkPlan] { - override def shift(node: SparkPlan): Iterable[SparkPlan] = { - val out = List(node) - .filter(cond.apply) - .flatMap(rule.shift) - out - } - override def shape(): Shape[SparkPlan] = rule.shape() - } - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index dfc2d474f7f3..92d64abf39c9 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -118,10 +118,7 @@ class EnumeratedApplier(session: SparkSession) (_: SparkSession) => FallbackEmptySchemaRelation() ) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarValidationRules() ::: - List( - (spark: SparkSession) => MergeTwoPhasesHashBaseAggregate(spark), - (_: SparkSession) => RewriteSparkPlanRulesManager() - ) ::: + List((spark: SparkSession) => MergeTwoPhasesHashBaseAggregate(spark)) ::: List( (session: SparkSession) => EnumeratedTransform(session, outputsColumnar), (_: SparkSession) => RemoveTransitions diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index 973020438370..dc34bc1af2a4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -16,13 +16,10 @@ */ package org.apache.gluten.extension.columnar.enumerated -import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{TransformExchange, TransformJoin, TransformOthers, TransformSingleNode} -import org.apache.gluten.extension.columnar.validator.{Validator, Validators} +import org.apache.gluten.extension.columnar.{OffloadExchange, OffloadJoin, OffloadOthers, OffloadSingleNode} import org.apache.gluten.planner.GlutenOptimization import org.apache.gluten.planner.property.Conventions import org.apache.gluten.ras.property.PropertySet -import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} import org.apache.gluten.utils.LogLevelUtil import org.apache.spark.sql.SparkSession @@ -34,31 +31,22 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) with LogLevelUtil { import EnumeratedTransform._ - private val validator = Validators - .builder() - .fallbackByHint() - .fallbackIfScanOnly() - .fallbackComplexExpressions() - .fallbackByBackendSettings() - .fallbackByUserOptions() - .build() - private val rules = List( - PushFilterToScan, - FilterRemoveRule + new PushFilterToScan(RasOffload.validator), + RemoveFilter ) // TODO: Should obey ReplaceSingleNode#applyScanNotTransformable to select // (vanilla) scan with cheaper sub-query plan through cost model. - private val implRules = List( - AsRasImplement(TransformOthers()), - AsRasImplement(TransformExchange()), - AsRasImplement(TransformJoin()), - ImplementAggregate, - ImplementFilter - ).map(_.withValidator(validator)) + private val offloadRules = List( + new AsRasOffload(OffloadOthers()), + new AsRasOffload(OffloadExchange()), + new AsRasOffload(OffloadJoin()), + RasOffloadAggregate, + RasOffloadFilter + ) - private val optimization = GlutenOptimization(rules ++ implRules) + private val optimization = GlutenOptimization(rules ++ offloadRules) private val reqConvention = Conventions.ANY private val altConventions = @@ -75,24 +63,12 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) } object EnumeratedTransform { - private case class AsRasImplement(delegate: TransformSingleNode) extends RasRule[SparkPlan] { - override def shift(node: SparkPlan): Iterable[SparkPlan] = { - val out = delegate.impl(node) - out match { - case t: GlutenPlan if !t.doValidate().isValid => - List.empty - case other => - List(other) - } - } - - override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) - } - // TODO: Currently not in use. Prepared for future development. - implicit private class RasRuleImplicits(rasRule: RasRule[SparkPlan]) { - def withValidator(v: Validator): RasRule[SparkPlan] = { - ConditionedRule.wrap(rasRule, v) + /** Accepts a [[OffloadSingleNode]] rule to convert it into a RAS offload rule. */ + private class AsRasOffload(delegate: OffloadSingleNode) extends RasOffload { + override protected def offload(node: SparkPlan): SparkPlan = { + val out = delegate.offload(node) + out } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala index f04f572c1ac9..7306b734a1d3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala @@ -17,28 +17,31 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.execution.{FilterHandler, TransformSupport} -import org.apache.gluten.extension.columnar.TransformHints +import org.apache.gluten.extension.columnar.validator.Validator import org.apache.gluten.ras.path.Pattern._ import org.apache.gluten.ras.path.Pattern.Matchers._ import org.apache.gluten.ras.rule.{RasRule, Shape} import org.apache.gluten.ras.rule.Shapes._ -import org.apache.spark.sql.execution.{ColumnarToRowExec, ColumnarToRowTransition, FileSourceScanExec, FilterExec, SparkPlan} +import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -object PushFilterToScan extends RasRule[SparkPlan] { +// TODO: Match on Vanilla filter + Gluten scan. +class PushFilterToScan(validator: Validator) extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { case FilterAndScan(filter, scan) => - if (!TransformHints.isTransformable(scan)) { - return List.empty - } - val newScan = - FilterHandler.pushFilterToScan(filter.condition, scan) - newScan match { - case ts: TransformSupport if ts.doValidate().isValid => - List(filter.withNewChildren(List(ts))) - case _ => + validator.validate(scan) match { + case Validator.Failed(reason) => List.empty + case Validator.Passed => + val newScan = + FilterHandler.pushFilterToScan(filter.condition, scan) + newScan match { + case ts: TransformSupport if ts.doValidate().isValid => + List(filter.withNewChildren(List(ts))) + case _ => + List.empty + } } case _ => List.empty diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala new file mode 100644 index 000000000000..57e093bdea53 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.enumerated + +import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode +import org.apache.gluten.extension.columnar.validator.{Validator, Validators} +import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} + +import org.apache.spark.sql.execution.SparkPlan + +trait RasOffload extends RasRule[SparkPlan] { + import RasOffload._ + + final override def shift(node: SparkPlan): Iterable[SparkPlan] = { + // 0. If the node is already offloaded, return fast. + if (node.isInstanceOf[GlutenPlan]) { + return List.empty + } + + // 1. Rewrite the node to form that native library supports. + val rewritten = rewrites.foldLeft(node) { + case (node, rewrite) => + node.transformUp { + case p => + val out = rewrite.rewrite(p) + out + } + } + + // 2. Walk the rewritten tree. + val offloaded = rewritten.transformUp { + case from => + // 3. Validate current node. If passed, offload it. + validator.validate(from) match { + case Validator.Passed => + offload(from) match { + case t: GlutenPlan if !t.doValidate().isValid => + // 4. If native validation fails on the offloaded node, return the + // original one. + from + case other => + other + } + case Validator.Failed(reason) => + from + } + } + + // 5. Return the final tree. + List(offloaded) + } + + protected def offload(node: SparkPlan): SparkPlan + + final override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) +} + +object RasOffload { + val validator = Validators + .builder() + .fallbackByHint() + .fallbackIfScanOnly() + .fallbackComplexExpressions() + .fallbackByBackendSettings() + .fallbackByUserOptions() + .build() + + private val rewrites = RewriteSingleNode.allRules() +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementAggregate.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadAggregate.scala similarity index 50% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementAggregate.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadAggregate.scala index 8c51ca4fd6cd..e48545ae9499 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementAggregate.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadAggregate.scala @@ -16,39 +16,16 @@ */ package org.apache.gluten.extension.columnar.enumerated -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.HashAggregateExecBaseTransformer -import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.HashAggregateExec -object ImplementAggregate extends RasRule[SparkPlan] { - override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { - case agg: HashAggregateExec => shiftAgg(agg) - case _ => List.empty +object RasOffloadAggregate extends RasOffload { + override protected def offload(node: SparkPlan): SparkPlan = node match { + case agg: HashAggregateExec => + val out = HashAggregateExecBaseTransformer.from(agg)() + out + case other => other } - - private def shiftAgg(agg: HashAggregateExec): Iterable[SparkPlan] = { - val transformer = implement(agg) - if (!transformer.doValidate().isValid) { - return List.empty - } - List(transformer) - } - - private def implement(agg: HashAggregateExec): HashAggregateExecBaseTransformer = { - BackendsApiManager.getSparkPlanExecApiInstance - .genHashAggregateExecTransformer( - agg.requiredChildDistributionExpressions, - agg.groupingExpressions, - agg.aggregateExpressions, - agg.aggregateAttributes, - agg.initialInputBufferOffset, - agg.resultExpressions, - agg.child - ) - } - - override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala similarity index 75% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementFilter.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala index 33121e7f1042..030d05d478f3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala @@ -17,22 +17,16 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} import org.apache.spark.sql.execution.{FilterExec, SparkPlan} -object ImplementFilter extends RasRule[SparkPlan] { - override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { +object RasOffloadFilter extends RasOffload { + override protected def offload(node: SparkPlan): SparkPlan = node match { case FilterExec(condition, child) => val out = BackendsApiManager.getSparkPlanExecApiInstance .genFilterExecTransformer(condition, child) - if (!out.doValidate().isValid) { - List.empty - } else { - List(out) - } - case _ => - List.empty + out + case other => + other } - override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/FilterRemoveRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala similarity index 97% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/FilterRemoveRule.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index 52b5be981194..c9f4b27bf203 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/FilterRemoveRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.execution.SparkPlan // because the pushed filter is not considered in the model. Removing the filter will make // optimizer choose a single scan as the winner sub-plan since a single scan's cost is lower than // filter + scan. -object FilterRemoveRule extends RasRule[SparkPlan] { +object RemoveFilter extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = { val filter = node.asInstanceOf[FilterExecTransformerBase] if (filter.isNoop()) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index d33cda2e6487..0e905ced11db 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -20,6 +20,7 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPostOverrides, TransformPreOverrides} +import org.apache.gluten.extension.columnar.rewrite.RewriteSparkPlanRulesManager import org.apache.gluten.extension.columnar.util.AdaptiveContext import org.apache.gluten.metrics.GlutenTimeMetric import org.apache.gluten.utils.{LogLevelUtil, PhysicalPlanSelector} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/PullOutPostProject.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPostProject.scala similarity index 92% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/PullOutPostProject.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPostProject.scala index dc2e6423cb45..1b546714447f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/PullOutPostProject.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPostProject.scala @@ -14,13 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension.columnar +package org.apache.gluten.extension.columnar.rewrite import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.utils.PullOutProjectHelper import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, NamedExpression, WindowExpression} -import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{GenerateExec, ProjectExec, SparkPlan} import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.execution.window.WindowExec @@ -33,17 +32,17 @@ import scala.collection.mutable.ArrayBuffer * the output of Spark, ensuring that the output data of the native plan can match the Spark plan * when a fallback occurs. */ -object PullOutPostProject extends Rule[SparkPlan] with PullOutProjectHelper { +object PullOutPostProject extends RewriteSingleNode with PullOutProjectHelper { private def needsPostProjection(plan: SparkPlan): Boolean = { plan match { case agg: BaseAggregateExec => val pullOutHelper = BackendsApiManager.getSparkPlanExecApiInstance.genHashAggregateExecPullOutHelper( - agg.groupingExpressions, agg.aggregateExpressions, agg.aggregateAttributes) - val allAggregateResultAttributes = pullOutHelper.allAggregateResultAttributes + val allAggregateResultAttributes = + pullOutHelper.allAggregateResultAttributes(agg.groupingExpressions) // If the result expressions has different size with output attribute, // post-projection is needed. agg.resultExpressions.size != allAggregateResultAttributes.size || @@ -72,14 +71,13 @@ object PullOutPostProject extends Rule[SparkPlan] with PullOutProjectHelper { } } - override def apply(plan: SparkPlan): SparkPlan = plan match { + override def rewrite(plan: SparkPlan): SparkPlan = plan match { case agg: BaseAggregateExec if supportedAggregate(agg) && needsPostProjection(agg) => val pullOutHelper = BackendsApiManager.getSparkPlanExecApiInstance.genHashAggregateExecPullOutHelper( - agg.groupingExpressions, agg.aggregateExpressions, agg.aggregateAttributes) - val newResultExpressions = pullOutHelper.allAggregateResultAttributes + val newResultExpressions = pullOutHelper.allAggregateResultAttributes(agg.groupingExpressions) val newAgg = copyBaseAggregateExec(agg)(newResultExpressions = newResultExpressions) ProjectExec(agg.resultExpressions, newAgg) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/PullOutPreProject.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala similarity index 96% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/PullOutPreProject.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala index 48a9a7687e4c..64d4f273622c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/PullOutPreProject.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension.columnar +package org.apache.gluten.extension.columnar.rewrite import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.sql.shims.SparkShimLoader @@ -22,8 +22,7 @@ import org.apache.gluten.utils.PullOutProjectHelper import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, Partial} -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.execution.{ExpandExec, GenerateExec, ProjectExec, SortExec, SparkPlan, TakeOrderedAndProjectExec} +import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, TypedAggregateExpression} import org.apache.spark.sql.execution.window.{WindowExec, WindowGroupLimitExecShim} @@ -36,7 +35,7 @@ import scala.collection.mutable * to transform the SparkPlan at the physical plan level, constructing a SparkPlan that supports * execution by the native engine. */ -object PullOutPreProject extends Rule[SparkPlan] with PullOutProjectHelper { +object PullOutPreProject extends RewriteSingleNode with PullOutProjectHelper { private def needsPreProject(plan: SparkPlan): Boolean = { plan match { @@ -118,7 +117,7 @@ object PullOutPreProject extends Rule[SparkPlan] with PullOutProjectHelper { } } - override def apply(plan: SparkPlan): SparkPlan = plan match { + override def rewrite(plan: SparkPlan): SparkPlan = plan match { case sort: SortExec if needsPreProject(sort) => val expressionMap = new mutable.HashMap[Expression, NamedExpression]() val newSortOrder = getNewSortOrder(sort.sortOrder, expressionMap) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteCollect.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala similarity index 93% rename from gluten-core/src/main/scala/org/apache/gluten/extension/RewriteCollect.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala index 3b6710857c94..74d493de5272 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteCollect.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension +package org.apache.gluten.extension.columnar.rewrite import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.utils.PullOutProjectHelper import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeSet, If, IsNotNull, IsNull, Literal, NamedExpression} -import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, CollectSet, Complete, Final, Partial} -import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.types.ArrayType @@ -36,7 +35,7 @@ import scala.collection.mutable.ArrayBuffer * * TODO: remove this rule once Velox compatible with vanilla Spark. */ -object RewriteCollect extends Rule[SparkPlan] with PullOutProjectHelper { +object RewriteCollect extends RewriteSingleNode with PullOutProjectHelper { private lazy val shouldRewriteCollect = BackendsApiManager.getSettings.shouldRewriteCollect() @@ -121,7 +120,7 @@ object RewriteCollect extends Rule[SparkPlan] with PullOutProjectHelper { (newAggregateAttributes, newResultExpressions) } - override def apply(plan: SparkPlan): SparkPlan = { + override def rewrite(plan: SparkPlan): SparkPlan = { if (!shouldRewriteCollect) { return plan } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteMultiChildrenCount.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteMultiChildrenCount.scala similarity index 93% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteMultiChildrenCount.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteMultiChildrenCount.scala index 9657c127da20..b395d961a075 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteMultiChildrenCount.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteMultiChildrenCount.scala @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension.columnar +package org.apache.gluten.extension.columnar.rewrite import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.utils.PullOutProjectHelper import org.apache.spark.sql.catalyst.expressions.{If, IsNull, Literal, Or} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Partial} -import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.types.IntegerType @@ -46,7 +45,7 @@ import org.apache.spark.sql.types.IntegerType * * TODO: Remove this rule when Velox support multi-children Count */ -object RewriteMultiChildrenCount extends Rule[SparkPlan] with PullOutProjectHelper { +object RewriteMultiChildrenCount extends RewriteSingleNode with PullOutProjectHelper { private lazy val shouldRewriteCount = BackendsApiManager.getSettings.shouldRewriteCount() private def extractCountForRewrite(aggExpr: AggregateExpression): Option[Count] = { @@ -92,7 +91,7 @@ object RewriteMultiChildrenCount extends Rule[SparkPlan] with PullOutProjectHelp } } - override def apply(plan: SparkPlan): SparkPlan = { + override def rewrite(plan: SparkPlan): SparkPlan = { if (!shouldRewriteCount) { return plan } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala new file mode 100644 index 000000000000..73bc8b967fad --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.rewrite + +import org.apache.gluten.extension.RewriteIn + +import org.apache.spark.sql.execution.SparkPlan + +/** + * Rewrites a plan node from vanilla Spark into its alternative representation. + * + * Gluten's planner will pick one that is considered the best executable plan between input plan and + * the output plan. + * + * Note: Only the current plan node is supposed to be open to modification. Do not access or modify + * the children node. Tree-walking is done by caller of this trait. + * + * TODO: Ideally for such API we'd better to allow multiple alternative outputs. + */ +trait RewriteSingleNode { + def rewrite(plan: SparkPlan): SparkPlan +} + +object RewriteSingleNode { + def allRules(): Seq[RewriteSingleNode] = { + Seq( + RewriteIn, + RewriteMultiChildrenCount, + RewriteCollect, + RewriteTypedImperativeAggregate, + PullOutPreProject, + PullOutPostProject) + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteSparkPlanRulesManager.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala similarity index 91% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteSparkPlanRulesManager.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala index 6070613c1ee7..5fd728eca65a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteSparkPlanRulesManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension.columnar +package org.apache.gluten.extension.columnar.rewrite -import org.apache.gluten.extension.{RewriteCollect, RewriteIn} +import org.apache.gluten.extension.columnar.{AddTransformHintRule, TransformHint, TransformHints} import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.rdd.RDD @@ -44,7 +44,7 @@ case class RewrittenNodeWall(originalChild: SparkPlan) extends LeafExecNode { * * Note that, this rule does not touch and tag these operators who does not need to rewrite. */ -class RewriteSparkPlanRulesManager private (rewriteRules: Seq[Rule[SparkPlan]]) +class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode]) extends Rule[SparkPlan] { private def mayNeedRewrite(plan: SparkPlan): Boolean = { @@ -83,7 +83,7 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[Rule[SparkPlan]]) // Some rewrite rules may generate new parent plan node, we should use transform to // rewrite the original plan. For example, PullOutPreProject and PullOutPostProject // will generate post-project plan node. - plan.transformUp { case p => rule.apply(p) } + plan.transformUp { case p => rule.rewrite(p) } } (rewrittenPlan, None) } catch { @@ -133,13 +133,6 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[Rule[SparkPlan]]) object RewriteSparkPlanRulesManager { def apply(): Rule[SparkPlan] = { - val rewriteRules = Seq( - RewriteIn, - RewriteMultiChildrenCount, - RewriteCollect, - RewriteTypedImperativeAggregate, - PullOutPreProject, - PullOutPostProject) - new RewriteSparkPlanRulesManager(rewriteRules) + new RewriteSparkPlanRulesManager(RewriteSingleNode.allRules()) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteTypedImperativeAggregate.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala similarity index 91% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteTypedImperativeAggregate.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala index df5341373d85..971a87923b23 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RewriteTypedImperativeAggregate.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala @@ -14,18 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension.columnar +package org.apache.gluten.extension.columnar.rewrite import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.utils.PullOutProjectHelper import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.BaseAggregateExec -object RewriteTypedImperativeAggregate extends Rule[SparkPlan] with PullOutProjectHelper { +object RewriteTypedImperativeAggregate extends RewriteSingleNode with PullOutProjectHelper { private lazy val shouldRewriteTypedImperativeAggregate = BackendsApiManager.getSettings.shouldRewriteTypedImperativeAggregate() @@ -40,7 +39,7 @@ object RewriteTypedImperativeAggregate extends Rule[SparkPlan] with PullOutProje } } - override def apply(plan: SparkPlan): SparkPlan = { + override def rewrite(plan: SparkPlan): SparkPlan = { if (!shouldRewriteTypedImperativeAggregate) { return plan } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala index a5b66df46b2e..2920c0a39819 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.planner.cost -import org.apache.gluten.extension.columnar.{ColumnarTransitions, TransformJoin} +import org.apache.gluten.extension.columnar.{ColumnarTransitions, OffloadJoin} import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Cost, CostModel} import org.apache.gluten.utils.PlanUtil @@ -57,7 +57,9 @@ object GlutenCostModel { // A very rough estimation as of now. private def selfLongCostOf(node: SparkPlan): Long = { node match { - case p: ShuffledHashJoinExec if !TransformJoin.isLegal(p) => + case p: ShuffledHashJoinExec if !OffloadJoin.isLegal(p) => + // To exclude the rewritten intermediate plan that is not executable + // by vanilla Spark and was generated by strategy "JoinSelectionOverrides" infLongCost case ColumnarToRowExec(child) => 3L case RowToColumnarExec(child) => 3L diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala index 7308703e7480..fbdbeadba886 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala @@ -18,8 +18,9 @@ package org.apache.spark.sql.execution.datasources import org.apache.gluten.execution.{ProjectExecTransformer, SortExecTransformer, TransformSupport, WholeStageTransformer} import org.apache.gluten.execution.datasource.GlutenFormatWriterInjects -import org.apache.gluten.extension.columnar.{AddTransformHintRule, RewriteSparkPlanRulesManager} +import org.apache.gluten.extension.columnar.AddTransformHintRule import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPreOverrides +import org.apache.gluten.extension.columnar.rewrite.RewriteSparkPlanRulesManager import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession From b96ddb4a1fd1e2bbb65473ea24c02d30f3e77fd9 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Tue, 7 May 2024 16:27:29 +0800 Subject: [PATCH 012/402] [GLUTEN-5580][CH]Fix cast to int exceed max (#5581) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #5580) How was this patch tested? test by ut --- ...enClickHouseTPCHSaltNullParquetSuite.scala | 12 +++++---- .../Functions/SparkFunctionCastFloatToInt.cpp | 27 ++++++++++--------- .../Functions/SparkFunctionCastFloatToInt.h | 12 ++++----- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index cc2eebcab6b6..866f0ffaaefa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2094,13 +2094,15 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr compareResultsAgainstVanillaSpark(sql, true, { _ => }) } - test("GLUTEN-3149: Fix convert exception of Inf to int") { - val tbl_create_sql = "create table test_tbl_3149(a int, b int) using parquet"; - val tbl_insert_sql = "insert into test_tbl_3149 values(1, 0)" - val select_sql = "select cast(a * 1.0f/b as int) as x from test_tbl_3149 where a = 1" + test("GLUTEN-3149/GLUTEN-5580: Fix convert float to int") { + val tbl_create_sql = "create table test_tbl_3149(a int, b bigint) using parquet"; + val tbl_insert_sql = "insert into test_tbl_3149 values(1, 0), (2, 171396196666200)" + val select_sql_1 = "select cast(a * 1.0f/b as int) as x from test_tbl_3149 where a = 1" + val select_sql_2 = "select cast(b/100 as int) from test_tbl_3149 where a = 2" spark.sql(tbl_create_sql) spark.sql(tbl_insert_sql); - compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) + compareResultsAgainstVanillaSpark(select_sql_1, true, { _ => }) + compareResultsAgainstVanillaSpark(select_sql_2, true, { _ => }) spark.sql("drop table test_tbl_3149") } diff --git a/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.cpp b/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.cpp index 322f9c08a243..c378f9fbf749 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.cpp @@ -15,7 +15,10 @@ * limitations under the License. */ +#include #include +#include +#include #include using namespace DB; @@ -36,18 +39,18 @@ struct NameToInt64 { static constexpr auto name = "sparkCastFloatToInt64"; }; struct NameToInt128 { static constexpr auto name = "sparkCastFloatToInt128"; }; struct NameToInt256 { static constexpr auto name = "sparkCastFloatToInt256"; }; -using SparkFunctionCastFloatToInt8 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToInt16 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToInt32 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToInt64 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToInt128 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToInt256 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToUInt8 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToUInt16 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToUInt32 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToUInt64 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToUInt128 = local_engine::SparkFunctionCastFloatToInt; -using SparkFunctionCastFloatToUInt256 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToInt8 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToInt16 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToInt32 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToInt64 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToInt128 = local_engine::SparkFunctionCastFloatToInt::max(), std::numeric_limits::min()>; +using SparkFunctionCastFloatToInt256 = local_engine::SparkFunctionCastFloatToInt::max(), std::numeric_limits::min()>; +using SparkFunctionCastFloatToUInt8 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToUInt16 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToUInt32 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToUInt64 = local_engine::SparkFunctionCastFloatToInt; +using SparkFunctionCastFloatToUInt128 = local_engine::SparkFunctionCastFloatToInt::max(), 0>; +using SparkFunctionCastFloatToUInt256 = local_engine::SparkFunctionCastFloatToInt::max(), 0>; REGISTER_FUNCTION(SparkFunctionCastToInt) { diff --git a/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.h b/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.h index 675db9e30fcd..4522e0e7d5f0 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionCastFloatToInt.h @@ -40,7 +40,7 @@ namespace ErrorCodes namespace local_engine { -template +template class SparkFunctionCastFloatToInt : public DB::IFunction { public: @@ -74,7 +74,7 @@ class SparkFunctionCastFloatToInt : public DB::IFunction DB::ColumnPtr src_col = arguments[0].column; size_t size = src_col->size(); - auto res_col = DB::ColumnVector::create(size); + auto res_col = DB::ColumnVector::create(size, 0); auto null_map_col = DB::ColumnUInt8::create(size, 0); switch(removeNullable(arguments[0].type)->getTypeId()) @@ -101,15 +101,15 @@ class SparkFunctionCastFloatToInt : public DB::IFunction { F element = src_vec->getElement(i); if (isNaN(element) || !isFinite(element)) - { - data[i] = 0; null_map_data[i] = 1; - } + else if (element > int_max_value) + data[i] = int_max_value; + else if (element < int_min_value) + data[i] = int_min_value; else data[i] = static_cast(element); } } - }; } \ No newline at end of file From eaa2761eb5f8ee6dcb557f8aa41eeb9fce8ac72a Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Tue, 7 May 2024 16:33:34 +0800 Subject: [PATCH 013/402] [GLUTEN-5622] Add new added Spark3.5 UTs in Gluten (#5623) In Spark 3.5, there are some new added UTs vs Spark3.4 which haven't been added in Gluten yet. In this PR, we ported the ones under spark/sql but excluding folders like streaming, execution/command, execution/benchmark. --- .../utils/velox/VeloxTestSettings.scala | 19 +++++++++++++++ .../GlutenBitmapExpressionsQuerySuite.scala | 21 +++++++++++++++++ .../apache/spark/sql/GlutenEmptyInSuite.scala | 19 +++++++++++++++ .../sql/GlutenRuntimeNullChecksV2Writes.scala | 19 +++++++++++++++ ...utenTableOptionsConstantFoldingSuite.scala | 21 +++++++++++++++++ .../GlutenDeltaBasedMergeIntoTableSuite.scala | 23 +++++++++++++++++++ ...ntoTableUpdateAsDeleteAndInsertSuite.scala | 23 +++++++++++++++++++ ...sedUpdateAsDeleteAndInsertTableSuite.scala | 23 +++++++++++++++++++ .../GlutenDeltaBasedUpdateTableSuite.scala | 23 +++++++++++++++++++ .../GlutenGroupBasedMergeIntoTableSuite.scala | 23 +++++++++++++++++++ ...emoveRedundantWindowGroupLimitsSuite.scala | 23 +++++++++++++++++++ ...nFileSourceCustomMetadataStructSuite.scala | 23 +++++++++++++++++++ .../GlutenTableLocationSuite.scala | 21 +++++++++++++++++ ...rquetFileMetadataStructRowIndexSuite.scala | 23 +++++++++++++++++++ 14 files changed, 304 insertions(+) create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenEmptyInSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenTableOptionsConstantFoldingSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateTableSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedMergeIntoTableSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenRemoveRedundantWindowGroupLimitsSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCustomMetadataStructSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenTableLocationSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileMetadataStructRowIndexSuite.scala diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e6a50a56a028..de5e4032e938 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1165,6 +1165,25 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenHiveSQLQuerySuite] enableSuite[GlutenCollapseProjectExecTransformerSuite] enableSuite[GlutenSparkSessionExtensionSuite] + enableSuite[GlutenBitmapExpressionsQuerySuite] + enableSuite[GlutenEmptyInSuite] + enableSuite[GlutenRuntimeNullChecksV2Writes] + enableSuite[GlutenTableOptionsConstantFoldingSuite] + enableSuite[GlutenDeltaBasedMergeIntoTableSuite] + enableSuite[GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite] + enableSuite[GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite] + // FIXME: complex type result mismatch + .exclude("update nested struct fields") + .exclude("update char/varchar columns") + enableSuite[GlutenDeltaBasedUpdateTableSuite] + enableSuite[GlutenGroupBasedMergeIntoTableSuite] + enableSuite[GlutenFileSourceCustomMetadataStructSuite] + enableSuite[GlutenParquetFileMetadataStructRowIndexSuite] + // Row index metadata column support in Velox isn't ready yet, refer velox-9147 + .exclude("reading _tmp_metadata_row_index - not present in a table") + .exclude("reading _tmp_metadata_row_index - present in a table") + enableSuite[GlutenTableLocationSuite] + enableSuite[GlutenRemoveRedundantWindowGroupLimitsSuite] override def getSQLQueryTestSettings: SQLQueryTestSettings = VeloxSQLQueryTestSettings } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala new file mode 100644 index 000000000000..e07821857a50 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenBitmapExpressionsQuerySuite + extends BitmapExpressionsQuerySuite + with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenEmptyInSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenEmptyInSuite.scala new file mode 100644 index 000000000000..ede561cbd6b1 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenEmptyInSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenEmptyInSuite extends EmptyInSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala new file mode 100644 index 000000000000..abd997bea82e --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenRuntimeNullChecksV2Writes.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenRuntimeNullChecksV2Writes extends RuntimeNullChecksV2Writes with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenTableOptionsConstantFoldingSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenTableOptionsConstantFoldingSuite.scala new file mode 100644 index 000000000000..35858433b406 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenTableOptionsConstantFoldingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenTableOptionsConstantFoldingSuite + extends TableOptionsConstantFoldingSuite + with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableSuite.scala new file mode 100644 index 000000000000..2ca5d06f9907 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenDeltaBasedMergeIntoTableSuite + extends DeltaBasedMergeIntoTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite.scala new file mode 100644 index 000000000000..47a3670d065f --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite + extends DeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite.scala new file mode 100644 index 000000000000..dd4f93140c48 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenDeltaBasedUpdateAsDeleteAndInsertTableSuite + extends DeltaBasedUpdateAsDeleteAndInsertTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateTableSuite.scala new file mode 100644 index 000000000000..b173c743a99f --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedUpdateTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenDeltaBasedUpdateTableSuite + extends DeltaBasedUpdateTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedMergeIntoTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedMergeIntoTableSuite.scala new file mode 100644 index 000000000000..9bf7abb2b70f --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedMergeIntoTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenGroupBasedMergeIntoTableSuite + extends GroupBasedMergeIntoTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenRemoveRedundantWindowGroupLimitsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenRemoveRedundantWindowGroupLimitsSuite.scala new file mode 100644 index 000000000000..9d819d2bd90f --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenRemoveRedundantWindowGroupLimitsSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenRemoveRedundantWindowGroupLimitsSuite + extends RemoveRedundantWindowGroupLimitsSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCustomMetadataStructSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCustomMetadataStructSuite.scala new file mode 100644 index 000000000000..9aed8f6d6541 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileSourceCustomMetadataStructSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenFileSourceCustomMetadataStructSuite + extends FileSourceCustomMetadataStructSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenTableLocationSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenTableLocationSuite.scala new file mode 100644 index 000000000000..879b0badf1dd --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenTableLocationSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenTableLocationSuite extends TableLocationSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileMetadataStructRowIndexSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileMetadataStructRowIndexSuite.scala new file mode 100644 index 000000000000..9d5291471b33 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFileMetadataStructRowIndexSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenParquetFileMetadataStructRowIndexSuite + extends ParquetFileMetadataStructRowIndexSuite + with GlutenSQLTestsBaseTrait {} From fe04e7711c84462cfba040997fd6dde1b2cd8125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Tue, 7 May 2024 20:45:30 +0800 Subject: [PATCH 014/402] [GLUTEN-4811][VL] Abfs FileSink Onboard (#5527) Support ABFS write --- cpp/velox/compute/VeloxRuntime.cc | 11 ++++ .../operators/writer/VeloxParquetDatasource.h | 7 +++ .../writer/VeloxParquetDatasourceABFS.h | 55 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index cfac1dadd2ab..8314d0bd271a 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -45,6 +45,10 @@ #include "operators/writer/VeloxParquetDatasourceGCS.h" #endif +#ifdef ENABLE_ABFS +#include "operators/writer/VeloxParquetDatasourceABFS.h" +#endif + using namespace facebook; namespace gluten { @@ -218,6 +222,13 @@ std::shared_ptr VeloxRuntime::createDatasource( #else throw std::runtime_error( "The write path is GCS path but the GCS haven't been enabled when writing parquet data in velox runtime!"); +#endif + } else if (isSupportedABFSPath(filePath)) { +#ifdef ENABLE_ABFS + return std::make_shared(filePath, veloxPool, sinkPool, schema); +#else + throw std::runtime_error( + "The write path is ABFS path but the ABFS haven't been enabled when writing parquet data in velox runtime!"); #endif } return std::make_shared(filePath, veloxPool, sinkPool, schema); diff --git a/cpp/velox/operators/writer/VeloxParquetDatasource.h b/cpp/velox/operators/writer/VeloxParquetDatasource.h index bf035c42355d..3df444016beb 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasource.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasource.h @@ -43,6 +43,9 @@ #include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h" #include "velox/connectors/hive/storage_adapters/hdfs/HdfsUtil.h" #endif +#ifdef ENABLE_ABFS +#include "velox/connectors/hive/storage_adapters/abfs/AbfsFileSystem.h" +#endif #include "velox/dwio/common/FileSink.h" #include "velox/dwio/common/Options.h" #include "velox/dwio/dwrf/reader/DwrfReader.h" @@ -72,6 +75,10 @@ inline bool isSupportedHDFSPath(const std::string& filePath) { return strncmp(filePath.c_str(), "hdfs:", 5) == 0; } +inline bool isSupportedABFSPath(const std::string& filePath) { + return strncmp(filePath.c_str(), "abfs:", 5) == 0 || strncmp(filePath.c_str(), "abfss:", 6) == 0; +} + class VeloxParquetDatasource : public Datasource { public: VeloxParquetDatasource( diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h new file mode 100644 index 000000000000..2251a46ffa69 --- /dev/null +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "operators/writer/VeloxParquetDatasource.h" +#include "utils/ConfigExtractor.h" +#include "utils/VeloxArrowUtils.h" + +#include + +#include "arrow/c/bridge.h" +#include "compute/VeloxRuntime.h" + +#include "velox/common/compression/Compression.h" +#include "velox/core/QueryConfig.h" +#include "velox/core/QueryCtx.h" +#include "velox/dwio/common/Options.h" + +namespace gluten { + +class VeloxParquetDatasourceABFS final : public VeloxParquetDatasource { + public: + VeloxParquetDatasourceABFS( + const std::string& filePath, + std::shared_ptr veloxPool, + std::shared_ptr sinkPool, + std::shared_ptr schema) + : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} + void init(const std::unordered_map& sparkConfs) override { + auto confs = std::make_shared(sparkConfs); + auto hiveConfs = getHiveConfig(confs); + auto fileSystem = filesystems::getFileSystem( + filePath_, std::make_shared(hiveConfs->valuesCopy())); + auto* abfsFileSystem = dynamic_cast(fileSystem.get()); + sink_ = std::make_unique( + abfsFileSystem->openFileForWrite(filePath_, {{}, sinkPool_.get()}), filePath_); + VeloxParquetDatasource::init(sparkConfs); + } +}; +} // namespace gluten From 254d62e7248c214591f8499ae2f99d1b669d9dfe Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Wed, 8 May 2024 00:53:57 +0800 Subject: [PATCH 015/402] [VL] Enable split preloading by default (#5456) Enable split preloading by default. use 1 I/O thread for each task thread --- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 3c7ddf32c71f..06d72ec57182 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -1230,9 +1230,14 @@ object GlutenConfig { val COLUMNAR_VELOX_CONNECTOR_IO_THREADS = buildStaticConf("spark.gluten.sql.columnar.backend.velox.IOThreads") .internal() - .doc("The IO threads for connector split preloading") + .doc("The Size of the IO thread pool in the Connector. This thread pool is used for split" + + " preloading and DirectBufferedInput.") .intConf - .createWithDefault(0) + .createWithDefaultFunction( + () => + SQLConf.get.getConfString("spark.executor.cores", "1").toInt / SQLConf.get + .getConfString("spark.task.cpus", "1") + .toInt) val COLUMNAR_VELOX_ASYNC_TIMEOUT = buildStaticConf("spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping") From 75fa35c46299d5361f997f9dce1545779beb6167 Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Wed, 8 May 2024 09:07:22 +0800 Subject: [PATCH 016/402] [GLUTEN-5603] Add new added Spark3.4 UTs in Gluten for Spark3.5 (#5637) Add the new added UTs in Spark3.4 for Gluten Spark35 profile --- .../utils/velox/VeloxTestSettings.scala | 11 +++++++++ .../sql/GlutenDataFrameToSchemaSuite.scala | 19 +++++++++++++++ .../spark/sql/GlutenDatasetUnpivotSuite.scala | 19 +++++++++++++++ .../sql/GlutenLateralColumnAliasSuite.scala | 19 +++++++++++++++ .../spark/sql/GlutenParametersSuite.scala | 19 +++++++++++++++ .../GlutenResolveDefaultColumnsSuite.scala | 21 +++++++++++++++++ .../GlutenSubqueryHintPropagationSuite.scala | 21 +++++++++++++++++ .../spark/sql/GlutenUrlFunctionsSuite.scala | 19 +++++++++++++++ ...GlutenDeltaBasedDeleteFromTableSuite.scala | 23 +++++++++++++++++++ ...GlutenGroupBasedDeleteFromTableSuite.scala | 23 +++++++++++++++++++ .../parquet/GlutenParquetRowIndexSuite.scala | 21 +++++++++++++++++ 11 files changed, 215 insertions(+) create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index de5e4032e938..7c809ec6dd60 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1165,6 +1165,17 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenHiveSQLQuerySuite] enableSuite[GlutenCollapseProjectExecTransformerSuite] enableSuite[GlutenSparkSessionExtensionSuite] + enableSuite[GlutenGroupBasedDeleteFromTableSuite] + enableSuite[GlutenDeltaBasedDeleteFromTableSuite] + enableSuite[GlutenDataFrameToSchemaSuite] + enableSuite[GlutenDatasetUnpivotSuite] + enableSuite[GlutenLateralColumnAliasSuite] + enableSuite[GlutenParametersSuite] + enableSuite[GlutenResolveDefaultColumnsSuite] + enableSuite[GlutenSubqueryHintPropagationSuite] + enableSuite[GlutenUrlFunctionsSuite] + // Row index metadata column support in Velox isn't ready yet, refer velox-9147 + // enableSuite[GlutenParquetRowIndexSuite] enableSuite[GlutenBitmapExpressionsQuerySuite] enableSuite[GlutenEmptyInSuite] enableSuite[GlutenRuntimeNullChecksV2Writes] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala new file mode 100644 index 000000000000..d578b92c4c8a --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameToSchemaSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenDataFrameToSchemaSuite extends DataFrameToSchemaSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala new file mode 100644 index 000000000000..e3ba780530fd --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDatasetUnpivotSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenDatasetUnpivotSuite extends DatasetUnpivotSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala new file mode 100644 index 000000000000..cc90f46e1a3d --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenLateralColumnAliasSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenLateralColumnAliasSuite extends LateralColumnAliasSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala new file mode 100644 index 000000000000..0887a7416fd8 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenParametersSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenParametersSuite extends ParametersSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala new file mode 100644 index 000000000000..2d1570be26c5 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenResolveDefaultColumnsSuite + extends ResolveDefaultColumnsSuite + with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala new file mode 100644 index 000000000000..323c5fbe1477 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSubqueryHintPropagationSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenSubqueryHintPropagationSuite + extends SubqueryHintPropagationSuite + with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala new file mode 100644 index 000000000000..ae173ecd47f9 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenUrlFunctionsSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class GlutenUrlFunctionsSuite extends UrlFunctionsSuite with GlutenSQLTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala new file mode 100644 index 000000000000..74893c5e51a2 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenDeltaBasedDeleteFromTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenDeltaBasedDeleteFromTableSuite + extends DeltaBasedDeleteFromTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala new file mode 100644 index 000000000000..25f377505c69 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/connector/GlutenGroupBasedDeleteFromTableSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connector + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenGroupBasedDeleteFromTableSuite + extends GroupBasedDeleteFromTableSuite + with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala new file mode 100644 index 000000000000..acf6a2b6384d --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.GlutenSQLTestsBaseTrait + +class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait {} From 01979372c950ab8d03e7eb1ea156de1838abdc7b Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 8 May 2024 10:13:45 +0800 Subject: [PATCH 017/402] [GLUTEN-4917][CI] Remove miniconda folder in image (#5646) --- .github/workflows/velox_velox_ut.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/velox_velox_ut.yml b/.github/workflows/velox_velox_ut.yml index ab2b07eb1c45..53a90cdc1720 100644 --- a/.github/workflows/velox_velox_ut.yml +++ b/.github/workflows/velox_velox_ut.yml @@ -46,6 +46,7 @@ jobs: mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | + rm -rf /opt/miniconda-for-velox/ cd ep/build-velox/src && \ ./get_velox.sh cd ../build/velox_ep/ From e975bf36881d031b8ef4e1b0c6eb5f1b675b895e Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Wed, 8 May 2024 10:34:21 +0800 Subject: [PATCH 018/402] [VL] Enable map_zip_with, zip_with functions (#5610) [VL] Enable map_zip_with, zip_with functions. --- .../gluten/utils/CHExpressionUtil.scala | 2 + .../ScalarFunctionsValidateSuite.scala | 37 +++++++++++++++++++ docs/velox-backend-support-progress.md | 3 +- .../expression/ExpressionMappings.scala | 2 + .../utils/velox/VeloxTestSettings.scala | 2 + .../sql/GlutenDataFrameFunctionsSuite.scala | 35 +++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 2 + .../sql/GlutenDataFrameFunctionsSuite.scala | 35 +++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 2 + .../sql/GlutenDataFrameFunctionsSuite.scala | 35 +++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 2 + .../sql/GlutenDataFrameFunctionsSuite.scala | 35 +++++++++++++++++- .../gluten/expression/ExpressionNames.scala | 2 + 13 files changed, 189 insertions(+), 5 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index 0e645d039840..b4190f1b8d8f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -186,6 +186,8 @@ object CHExpressionUtil { SKEWNESS -> DefaultValidator(), BIT_LENGTH -> DefaultValidator(), MAKE_YM_INTERVAL -> DefaultValidator(), + MAP_ZIP_WITH -> DefaultValidator(), + ZIP_WITH -> DefaultValidator(), KURTOSIS -> DefaultValidator(), REGR_R2 -> DefaultValidator(), REGR_SLOPE -> DefaultValidator(), diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 0a7a7d6cb9a6..200c5f55147d 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -517,6 +517,43 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("map_zip_with") { + withTempPath { + path => + Seq((Map("a" -> 1, "b" -> 2), Map("a" -> 2, "b" -> 3))) + .toDF("m1", "m2") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("map_tbl") + + runQueryAndCompare( + "select map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2) from map_tbl") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + + test("zip_with") { + withTempPath { + path => + Seq[(Seq[Integer], Seq[Integer])]( + (Seq(9001, 9002, 9003), Seq(4, 5, 6)), + (Seq(1, 2), Seq(3, 4)), + (Seq.empty, Seq.empty), + (null, null) + ).toDF("val1", "val2") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl") + + runQueryAndCompare("select zip_with(val1, val2, (x, y) -> x + y) from array_tbl") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("Test isnan function") { runQueryAndCompare( "SELECT isnan(l_orderkey), isnan(cast('NaN' as double)), isnan(0.0F/0.0F)" + diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index e98587efbbe4..1171b7d91d1a 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -291,6 +291,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | map_from_entries | map_from_entries | | | | | | | | | | | | | | | | | | | | | | | map_keys | map_keys | map_keys | S | | | | | | | | | | | | | | | | | | | | | map_values | map_values | map_values | S | | | | | | | | | | | | | | | | | S | | | +| map_zip_with | map_zip_with | | S | | | | | | | | | | | | | | | | | S | | | | named_struct,struct | row_construct | named_struct | S | | | | | | | | | | | | | | | | | | S | | | posexplode_outer,posexplode | | | | | | | | | | | | | | | | | | | | | | | | sequence | | | | | | | | | | | | | | | | | | | | | | | @@ -302,7 +303,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | transform | transform | transofrm | | | | | | | | | | | | | | | | | | | | | | transform_keys | transform_keys | | | | | | | | | | | | | | | | | | | | | | | transform_values | transform_values | | | | | | | | | | | | | | | | | | | | | | -| zip_with | zip_with | | | | | | | | | | | | | | | | | | | | | | +| zip_with | zip_with | S | | | | | | | | | | | | | | | | | | | | | | add_months | | | S | | | | | | | | | | | | | | | | | | | | | current_date | | | S* | | | | | | | | | | | | | | | | | | | | | current_timestamp | | | S* | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 920b6fab823e..ef43c2724dc5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -240,6 +240,7 @@ object ExpressionMappings { Sig[ArrayForAll](FORALL), Sig[ArrayExists](EXISTS), Sig[Shuffle](SHUFFLE), + Sig[ZipWith](ZIP_WITH), // Map functions Sig[CreateMap](CREATE_MAP), Sig[GetMapValue](GET_MAP_VALUE), @@ -247,6 +248,7 @@ object ExpressionMappings { Sig[MapValues](MAP_VALUES), Sig[MapFromArrays](MAP_FROM_ARRAYS), Sig[MapEntries](MAP_ENTRIES), + Sig[MapZipWith](MAP_ZIP_WITH), Sig[StringToMap](STR_TO_MAP), // Struct functions Sig[GetStructField](GET_STRUCT_FIELD), diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 4844b735198e..b9f8c066578a 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -278,6 +278,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("shuffle function - array for primitive type not containing null") .exclude("shuffle function - array for primitive type containing null") .exclude("shuffle function - array for non-primitive type") + // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. + .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameTungstenSuite] enableSuite[GlutenDataFrameSetOperationsSuite] // Result depends on the implementation for nondeterministic expression rand. diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala index 44981e1cee71..2b0b40790a76 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala @@ -16,4 +16,37 @@ */ package org.apache.spark.sql -class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.functions._ + +class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait { + import testImplicits._ + + testGluten("map_zip_with function - map of primitive types") { + val df = Seq( + (Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))), + (Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))), + (Map.empty[Int, Long], Map[Integer, Integer]((5, 1))), + (Map(5 -> 1L), null) + ).toDF("m1", "m2") + + GlutenQueryTestUtil.sameRows( + df.selectExpr("map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2)").collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + + GlutenQueryTestUtil.sameRows( + df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)).collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + } +} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index c6f6b0d3397d..642873028583 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -943,6 +943,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("shuffle function - array for primitive type not containing null") .exclude("shuffle function - array for primitive type containing null") .exclude("shuffle function - array for non-primitive type") + // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. + .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameHintSuite] enableSuite[GlutenDataFrameImplicitsSuite] enableSuite[GlutenDataFrameJoinSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala index 44981e1cee71..2b0b40790a76 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala @@ -16,4 +16,37 @@ */ package org.apache.spark.sql -class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.functions._ + +class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait { + import testImplicits._ + + testGluten("map_zip_with function - map of primitive types") { + val df = Seq( + (Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))), + (Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))), + (Map.empty[Int, Long], Map[Integer, Integer]((5, 1))), + (Map(5 -> 1L), null) + ).toDF("m1", "m2") + + GlutenQueryTestUtil.sameRows( + df.selectExpr("map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2)").collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + + GlutenQueryTestUtil.sameRows( + df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)).collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + } +} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 033e084b8522..a046f0a02a1e 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -948,6 +948,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("shuffle function - array for primitive type not containing null") .exclude("shuffle function - array for primitive type containing null") .exclude("shuffle function - array for non-primitive type") + // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. + .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameHintSuite] enableSuite[GlutenDataFrameImplicitsSuite] enableSuite[GlutenDataFrameJoinSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala index 44981e1cee71..2b0b40790a76 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala @@ -16,4 +16,37 @@ */ package org.apache.spark.sql -class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.functions._ + +class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait { + import testImplicits._ + + testGluten("map_zip_with function - map of primitive types") { + val df = Seq( + (Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))), + (Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))), + (Map.empty[Int, Long], Map[Integer, Integer]((5, 1))), + (Map(5 -> 1L), null) + ).toDF("m1", "m2") + + GlutenQueryTestUtil.sameRows( + df.selectExpr("map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2)").collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + + GlutenQueryTestUtil.sameRows( + df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)).collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + } +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 7c809ec6dd60..4583396b484d 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -964,6 +964,8 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("shuffle function - array for primitive type not containing null") .exclude("shuffle function - array for primitive type containing null") .exclude("shuffle function - array for non-primitive type") + // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. + .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameHintSuite] enableSuite[GlutenDataFrameImplicitsSuite] enableSuite[GlutenDataFrameJoinSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala index 44981e1cee71..2b0b40790a76 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala @@ -16,4 +16,37 @@ */ package org.apache.spark.sql -class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.functions._ + +class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenSQLTestsTrait { + import testImplicits._ + + testGluten("map_zip_with function - map of primitive types") { + val df = Seq( + (Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))), + (Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))), + (Map.empty[Int, Long], Map[Integer, Integer]((5, 1))), + (Map(5 -> 1L), null) + ).toDF("m1", "m2") + + GlutenQueryTestUtil.sameRows( + df.selectExpr("map_zip_with(m1, m2, (k, v1, v2) -> k == v1 + v2)").collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + + GlutenQueryTestUtil.sameRows( + df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)).collect.toSeq, + Seq( + Row(Map(8 -> true, 3 -> false, 6 -> true)), + Row(Map(10 -> null, 8 -> false, 4 -> null)), + Row(Map(5 -> null)), + Row(null)), + false + ) + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 2ad1c6f739c5..54a55b355f36 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -257,6 +257,7 @@ object ExpressionNames { final val EXISTS = "exists" final val TRANSFORM = "transform" final val SHUFFLE = "shuffle" + final val ZIP_WITH = "zip_with" // Map functions final val CREATE_MAP = "map" @@ -265,6 +266,7 @@ object ExpressionNames { final val MAP_VALUES = "map_values" final val MAP_FROM_ARRAYS = "map_from_arrays" final val MAP_ENTRIES = "map_entries" + final val MAP_ZIP_WITH = "map_zip_with" final val STR_TO_MAP = "str_to_map" // struct functions From e633887c8447cb136015da5a817898a759739697 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 8 May 2024 11:23:58 +0800 Subject: [PATCH 019/402] [VL] Add a bad test case that final aggregate of collect_list is fallen back while partial aggregate is not (#5649) --- .../gluten/execution/FallbackSuite.scala | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index 91024da1c4b1..fbad525a20dd 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -106,6 +106,40 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl } } + // java.lang.NullPointerException + ignore("fallback final aggregate of collect_list") { + withSQLConf( + GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1", + GlutenConfig.COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR.key -> "false", + GlutenConfig.EXPRESSION_BLACK_LIST.key -> "element_at" + ) { + runQueryAndCompare( + "SELECT sum(ele) FROM (SELECT c1, element_at(collect_list(c2), 1) as ele FROM tmp1 " + + "GROUP BY c1)") { + df => + val columnarToRow = collectColumnarToRow(df.queryExecution.executedPlan) + assert(columnarToRow == 1) + } + } + } + + // java.lang.NullPointerException + ignore("fallback final aggregate of collect_set") { + withSQLConf( + GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1", + GlutenConfig.COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR.key -> "false", + GlutenConfig.EXPRESSION_BLACK_LIST.key -> "element_at" + ) { + runQueryAndCompare( + "SELECT sum(ele) FROM (SELECT c1, element_at(collect_set(c2), 1) as ele FROM tmp1 " + + "GROUP BY c1)") { + df => + val columnarToRow = collectColumnarToRow(df.queryExecution.executedPlan) + assert(columnarToRow == 1) + } + } + } + test("fallback with AQE read") { runQueryAndCompare( """ From cee1f3bd869340c156feb0cdaf0a867f32343d54 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Wed, 8 May 2024 11:35:25 +0800 Subject: [PATCH 020/402] [GLUTEN-5414] [VL] Support Arrow native memory pool usage track (#5550) --- .../execution/RowToVeloxColumnarExec.scala | 2 +- .../python/ColumnarArrowEvalPythonExec.scala | 2 +- .../apache/gluten/utils/DatasourceUtil.scala | 2 +- .../ColumnarCachedBatchSerializer.scala | 2 +- .../VeloxColumnarWriteFilesExec.scala | 2 +- .../velox/VeloxFormatWriterInjects.scala | 2 +- ...VeloxCelebornColumnarBatchSerializer.scala | 2 +- gluten-data/pom.xml | 28 +++++++ .../gluten/columnarbatch/ColumnarBatches.java | 2 +- .../alloc}/ArrowBufferAllocators.java | 2 +- .../alloc}/ManagedAllocationListener.java | 2 +- .../arrow/pool/ArrowNativeMemoryPool.java | 75 +++++++++++++++++++ .../arrow/pool/ArrowReservationListener.java | 41 ++++++++++ .../vectorized/ArrowWritableColumnVector.java | 2 +- .../vectorized/ColumnarBatchInIterator.java | 2 +- .../apache/gluten/utils/ImplicitClass.scala | 2 +- .../vectorized/ColumnarBatchSerializer.scala | 2 +- .../execution/ColumnarBuildSideRelation.scala | 2 +- .../spark/sql/execution/utils/ExecUtil.scala | 2 +- .../spark/sql/utils/SparkVectorUtil.scala | 2 +- 20 files changed, 161 insertions(+), 17 deletions(-) rename gluten-data/src/main/java/org/apache/gluten/memory/{arrowalloc => arrow/alloc}/ArrowBufferAllocators.java (98%) rename gluten-data/src/main/java/org/apache/gluten/memory/{arrowalloc => arrow/alloc}/ManagedAllocationListener.java (98%) create mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java create mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 22ca020d1ac2..be1bc64e21b8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenException import org.apache.gluten.exec.Runtimes -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.{ArrowAbiUtil, Iterators} import org.apache.gluten.vectorized._ diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala index f2beef6ca6fb..77ef1c6422b2 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala @@ -18,7 +18,7 @@ package org.apache.spark.api.python import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.Iterators import org.apache.gluten.vectorized.ArrowWritableColumnVector diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala index f61cdb8e9986..6150507b4baa 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala @@ -17,7 +17,7 @@ package org.apache.gluten.utils import org.apache.gluten.datasource.DatasourceJniWrapper -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.spark.sql.types.StructType diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index 88678cb5e5c3..7385c53d61b3 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -21,7 +21,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exec.Runtimes import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.{ArrowAbiUtil, Iterators} import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala index 26d249f9056b..23dff990c464 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenException import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.{Partition, SparkException, TaskContext, TaskOutputFileAlreadyExistException} diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala index 97ebb932ac05..c358d6372c36 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala @@ -20,7 +20,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.datasource.DatasourceJniWrapper import org.apache.gluten.exception.GlutenException import org.apache.gluten.execution.datasource.GlutenRowSplitter -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.{ArrowAbiUtil, DatasourceUtil} diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index c5bd8874853a..d72977f59714 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -18,7 +18,7 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.exec.Runtimes -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.vectorized._ diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml index 951f53ee83d3..db617112f652 100644 --- a/gluten-data/pom.xml +++ b/gluten-data/pom.xml @@ -137,6 +137,34 @@ compile + + org.apache.arrow + arrow-dataset + ${arrow.version} + + + io.netty + netty-common + + + io.netty + netty-buffer + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + protobuf-java + com.google.protobuf + + + compile + diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java index 85ef875d69a2..624428dcba19 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java @@ -19,7 +19,7 @@ import org.apache.gluten.exception.GlutenException; import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.Runtimes; -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators; +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; import org.apache.gluten.memory.nmm.NativeMemoryManager; import org.apache.gluten.utils.ArrowAbiUtil; import org.apache.gluten.utils.ArrowUtil; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ArrowBufferAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java similarity index 98% rename from gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ArrowBufferAllocators.java rename to gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java index fcac26d6ebb1..efee20e48b83 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ArrowBufferAllocators.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.memory.arrowalloc; +package org.apache.gluten.memory.arrow.alloc; import org.apache.gluten.memory.memtarget.MemoryTargets; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ManagedAllocationListener.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java similarity index 98% rename from gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ManagedAllocationListener.java rename to gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java index ded2af60bd3a..a76c0aabee3b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/arrowalloc/ManagedAllocationListener.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.memory.arrowalloc; +package org.apache.gluten.memory.arrow.alloc; import org.apache.gluten.GlutenConfig; import org.apache.gluten.memory.SimpleMemoryUsageRecorder; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java new file mode 100644 index 000000000000..04a6e0002ade --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.memory.arrow.pool; + +import org.apache.arrow.dataset.jni.NativeMemoryPool; +import org.apache.spark.util.TaskResource; +import org.apache.spark.util.TaskResources; +import org.apache.spark.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ArrowNativeMemoryPool implements TaskResource { + private static final Logger LOGGER = LoggerFactory.getLogger(ArrowNativeMemoryPool.class); + + private final NativeMemoryPool arrowPool; + private final ArrowReservationListener listener; + + public ArrowNativeMemoryPool() { + listener = new ArrowReservationListener(TaskResources.getSharedUsage()); + arrowPool = NativeMemoryPool.createListenable(listener); + } + + public static NativeMemoryPool arrowPool(String name) { + if (!TaskResources.inSparkTask()) { + throw new IllegalStateException("This method must be called in a Spark task."); + } + String id = "ArrowNativeMemoryPool:" + name; + return TaskResources.addResourceIfNotRegistered(id, () -> createArrowNativeMemoryPool(name)) + .getArrowPool(); + } + + private static ArrowNativeMemoryPool createArrowNativeMemoryPool(String name) { + return new ArrowNativeMemoryPool(); + } + + @Override + public void release() throws Exception { + if (arrowPool.getBytesAllocated() != 0) { + LOGGER.warn( + String.format( + "Arrow pool still reserved non-zero bytes, " + + "which may cause memory leak, size: %s. ", + Utils.bytesToString(arrowPool.getBytesAllocated()))); + } + arrowPool.close(); + } + + @Override + public int priority() { + return 0; + } + + @Override + public String resourceName() { + return "arrow_mem"; + } + + public NativeMemoryPool getArrowPool() { + return arrowPool; + } +} diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java new file mode 100644 index 000000000000..2e1a254453f2 --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowReservationListener.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.memory.arrow.pool; + +import org.apache.gluten.memory.SimpleMemoryUsageRecorder; + +public class ArrowReservationListener implements org.apache.arrow.dataset.jni.ReservationListener { + private final SimpleMemoryUsageRecorder sharedUsage; // shared task metrics + + public ArrowReservationListener(SimpleMemoryUsageRecorder recorder) { + this.sharedUsage = recorder; + } + + @Override + public void reserve(long size) { + synchronized (this) { + sharedUsage.inc(size); + } + } + + @Override + public void unreserve(long size) { + synchronized (this) { + sharedUsage.inc(-size); + } + } +} diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java index 4603cbc4187d..dfd570debc0a 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ArrowWritableColumnVector.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators; +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.BigIntVector; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java index 85b24166ebf5..bd89f62a1806 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java @@ -18,7 +18,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatchJniWrapper; import org.apache.gluten.columnarbatch.ColumnarBatches; -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators; +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; import org.apache.spark.sql.vectorized.ColumnarBatch; diff --git a/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala b/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala index 1a5ae42777ff..4ffb3ab5cdcc 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala @@ -17,7 +17,7 @@ package org.apache.gluten.utils import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.sql.vectorized.ColumnarBatch diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index 9fbdb36377e3..e632700e3743 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -18,7 +18,7 @@ package org.apache.gluten.vectorized import org.apache.gluten.GlutenConfig import org.apache.gluten.exec.Runtimes -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.ArrowAbiUtil diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index ac94bfa89d67..9d9f5ab1765c 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exec.Runtimes -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.{ArrowAbiUtil, Iterators} diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index fd0cf45c8a2a..d3e7b409686a 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.utils import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.Iterators import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning} diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala index 208afd294920..3e86be79ac3f 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkVectorUtil.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.utils import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.arrowalloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.sql.vectorized.ColumnarBatch From 3f0bd06cfbe5b04a778167f66d471b3c67844c13 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Tue, 7 May 2024 23:30:10 -0500 Subject: [PATCH 021/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240508) (#5645) Auto commit by gluten daily build, please check the build status and merge it if it's green. need merge pr https://github.com/ClickHouse/ClickHouse/pull/62904 --- cpp-ch/clickhouse.version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index e3e9f01fe43d..9f4ee9241ba7 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240506 -CH_COMMIT=341de779fe0 \ No newline at end of file +CH_BRANCH=rebase_ch/20240508 +CH_COMMIT=1dfaf7ffeaa \ No newline at end of file From 071d891cf513c398f3065e36b6447391c9dda13f Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Wed, 8 May 2024 13:50:26 +0800 Subject: [PATCH 022/402] [GLUTEN-5352][GLUTEN-5459][CH]Fix and improve year function (#5455) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #5352, #5459) How was this patch tested? TEST BY UT 性能测试： 端到端SQL测试 表test_tbl(d string), 数据量 3000W行，测试SQL select count(1) from test_tbl where year(d) > '1990' PR改动前：5.245s， 5.442s， 5.458s； PR改动后：4.689s，4.782s， 5.011s; --- ...enClickHouseTPCHSaltNullParquetSuite.scala | 7 +- .../Functions/FunctionGetDateData.h | 173 ++++++++++++++++++ .../Functions/SparkFunctionExtractYear.cpp | 46 +++++ .../Functions/SparkFunctionToDate.cpp | 149 +-------------- .../Parser/SerializedPlanParser.cpp | 16 ++ 5 files changed, 244 insertions(+), 147 deletions(-) create mode 100644 cpp-ch/local-engine/Functions/FunctionGetDateData.h create mode 100644 cpp-ch/local-engine/Functions/SparkFunctionExtractYear.cpp diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 866f0ffaaefa..20638615d3c8 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2508,11 +2508,14 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("drop table test_tbl_4279") } - test("GLUTEN-4997: Bug fix year diff") { + test("GLUTEN-4997/GLUTEN-5352: Bug fix year diff") { val tbl_create_sql = "create table test_tbl_4997(id bigint, data string) using parquet" val tbl_insert_sql = "insert into test_tbl_4997 values(1, '2024-01-03'), (2, '2024'), (3, '2024-'), (4, '2024-1')," + - "(5, '2024-1-'), (6, '2024-1-3'), (7, '2024-1-3T'), (8, '21-0'), (9, '12-9')"; + "(5, '2024-1-'), (6, '2024-1-3'), (7, '2024-1-3T'), (8, '21-0'), (9, '12-9'), (10, '-1')," + + "(11, '999'), (12, '1000'), (13, '9999'), (15, '2024-04-19 00:00:00-12'), (16, '2024-04-19 00:00:00+12'), " + + "(17, '2024-04-19 23:59:59-12'), (18, '2024-04-19 23:59:59+12'), (19, '1899-12-01')," + + "(20, '2024:12'), (21, '2024ABC'), (22, NULL), (23, '0'), (24, '')" val select_sql = "select id, year(data) from test_tbl_4997 order by id" spark.sql(tbl_create_sql) spark.sql(tbl_insert_sql) diff --git a/cpp-ch/local-engine/Functions/FunctionGetDateData.h b/cpp-ch/local-engine/Functions/FunctionGetDateData.h new file mode 100644 index 000000000000..4f79d4bd0c4b --- /dev/null +++ b/cpp-ch/local-engine/Functions/FunctionGetDateData.h @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +template +class FunctionGetDateData : public DB::IFunction +{ +public: + FunctionGetDateData() = default; + ~FunctionGetDateData() override = default; + + DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t) const override + { + if (arguments.size() != 1) + throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s arguments number must be 1.", getName()); + + const DB::ColumnWithTypeAndName arg1 = arguments[0]; + const auto * src_col = checkAndGetColumn(arg1.column.get()); + size_t size = src_col->size(); + + if (!result_type->isNullable()) + throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s return type must be nullable", getName()); + + using ColVecTo = ColumnVector; + typename ColVecTo::MutablePtr result_column = ColVecTo::create(size, 0); + typename ColVecTo::Container & result_container = result_column->getData(); + DB::ColumnUInt8::MutablePtr null_map = DB::ColumnUInt8::create(size, 0); + typename DB::ColumnUInt8::Container & null_container = null_map->getData(); + const DateLUTImpl * local_time_zone = &DateLUT::instance(); + const DateLUTImpl * utc_time_zone = &DateLUT::instance("UTC"); + + for (size_t i = 0; i < size; ++i) + { + auto str = src_col->getDataAt(i); + if (str.size < 4) + { + null_container[i] = true; + continue; + } + else + { + DB::ReadBufferFromMemory buf(str.data, str.size); + while(!buf.eof() && *buf.position() == ' ') + { + buf.position() ++; + } + if(buf.buffer().end() - buf.position() < 4) + { + null_container[i] = true; + continue; + } + bool can_be_parsed = true; + if (!checkAndGetDateData(buf, buf.buffer().end() - buf.position(), result_container[i], *local_time_zone, can_be_parsed)) + { + if (!can_be_parsed) + null_container[i] = true; + else + { + time_t tmp = 0; + bool parsed = tryParseDateTimeBestEffort(tmp, buf, *local_time_zone, *utc_time_zone); + if (get_date) + result_container[i] = local_time_zone->toDayNum(tmp); + null_container[i] = !parsed; + } + } + } + } + return DB::ColumnNullable::create(std::move(result_column), std::move(null_map)); + } + +private: + bool checkAndGetDateData(DB::ReadBuffer & buf, size_t buf_size, T &x, const DateLUTImpl & date_lut, bool & can_be_parsed) const + { + auto checkNumbericASCII = [&](DB::ReadBuffer & rb, size_t start, size_t length) -> bool + { + for (size_t i = start; i < start + length; ++i) + { + if (i >= buf_size || !isNumericASCII(*(rb.position() + i))) + { + return false; + } + } + return true; + }; + auto checkDelimiter = [&](DB::ReadBuffer & rb, size_t pos) -> bool + { + if (pos >= buf_size || *(rb.position() + pos) != '-') + return false; + else + return true; + }; + bool yearNumberCanbeParsed = checkNumbericASCII(buf, 0, 4) && (buf_size == 4 || checkDelimiter(buf, 4)); + Int16 year = 0; + if (yearNumberCanbeParsed) + { + year = (*(buf.position() + 0) - '0') * 1000 + + (*(buf.position() + 1) - '0') * 100 + + (*(buf.position() + 2) - '0') * 10 + + (*(buf.position() + 3) - '0'); + x = get_year ? year : 0; + } + if (!yearNumberCanbeParsed + || !checkNumbericASCII(buf, 5, 2) + || !checkDelimiter(buf, 7) + || !checkNumbericASCII(buf, 8, 2)) + { + can_be_parsed = yearNumberCanbeParsed; + return false; + } + else + { + UInt8 month = (*(buf.position() + 5) - '0') * 10 + (*(buf.position() + 6) - '0'); + if (month <= 0 || month > 12) + return false; + UInt8 day = (*(buf.position() + 8) - '0') * 10 + (*(buf.position() + 9) - '0'); + if (day <= 0 || day > 31) + return false; + else if (day == 31 && (month == 2 || month == 4 || month == 6 || month == 9 || month == 11)) + return false; + else if (day == 30 && month == 2) + return false; + else + { + if (day == 29 && month == 2 && year % 4 != 0) + return false; + else + { + if (get_date) + x = date_lut.makeDayNum(year, month, day, -static_cast(date_lut.getDayNumOffsetEpoch())); + return true; + } + } + } + } +}; +} diff --git a/cpp-ch/local-engine/Functions/SparkFunctionExtractYear.cpp b/cpp-ch/local-engine/Functions/SparkFunctionExtractYear.cpp new file mode 100644 index 000000000000..3a88d32770f0 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkFunctionExtractYear.cpp @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +using namespace DB; + +namespace local_engine +{ +class SparkFunctionExtractYear : public FunctionGetDateData +{ +public: + static constexpr auto name = "sparkExtractYear"; + static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared(); } + SparkFunctionExtractYear() = default; + ~SparkFunctionExtractYear() override = default; + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + String getName() const override { return name; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + return makeNullable(std::make_shared()); + } +}; + +REGISTER_FUNCTION(SparkExtractYear) +{ + factory.registerFunction(); +} +} \ No newline at end of file diff --git a/cpp-ch/local-engine/Functions/SparkFunctionToDate.cpp b/cpp-ch/local-engine/Functions/SparkFunctionToDate.cpp index 1c5d68fcdd39..c527ca3ff5c9 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionToDate.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionToDate.cpp @@ -14,33 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} -} namespace local_engine { -class SparkFunctionConvertToDate : public DB::IFunction +class SparkFunctionConvertToDate : public FunctionGetDateData { public: static constexpr auto name = "sparkToDate"; @@ -53,130 +32,10 @@ class SparkFunctionConvertToDate : public DB::IFunction bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } - bool checkAndGetDate32(DB::ReadBuffer & buf, DB::DataTypeDate32::FieldType &x, const DateLUTImpl & date_lut, UInt8 & can_be_parsed) const - { - auto checkNumbericASCII = [&](DB::ReadBuffer & rb, size_t start, size_t length) -> bool - { - for (size_t i = start; i < start + length; ++i) - { - if (!isNumericASCII(*(rb.position() + i))) - { - return false; - } - } - return true; - }; - auto checkDelimiter = [&](DB::ReadBuffer & rb, size_t pos) -> bool - { - if (*(rb.position() + pos) != '-') - return false; - else - return true; - }; - bool yearIsNumberic = checkNumbericASCII(buf, 0, 4); - if (!yearIsNumberic - || !checkDelimiter(buf, 4) - || !checkNumbericASCII(buf, 5, 2) - || !checkDelimiter(buf, 7) - || !checkNumbericASCII(buf, 8, 2)) - { - can_be_parsed = yearIsNumberic; - return false; - } - else - { - UInt8 month = (*(buf.position() + 5) - '0') * 10 + (*(buf.position() + 6) - '0'); - if (month <= 0 || month > 12) - return false; - UInt8 day = (*(buf.position() + 8) - '0') * 10 + (*(buf.position() + 9) - '0'); - if (day <= 0 || day > 31) - return false; - else if (day == 31 && (month == 2 || month == 4 || month == 6 || month == 9 || month == 11)) - return false; - else if (day == 30 && month == 2) - return false; - else - { - Int16 year = (*(buf.position() + 0) - '0') * 1000 + - (*(buf.position() + 1) - '0') * 100 + - (*(buf.position() + 2) - '0') * 10 + - (*(buf.position() + 3) - '0'); - if (day == 29 && month == 2 && year % 4 != 0) - return false; - else - { - x = date_lut.makeDayNum(year, month, day, -static_cast(date_lut.getDayNumOffsetEpoch())); - return true; - } - } - } - } - DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName &) const override { - DB::DataTypePtr date32_type = std::make_shared(); - return makeNullable(date32_type); - } - - DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t) const override - { - if (arguments.size() != 1) - throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s arguments number must be 1.", name); - - const DB::ColumnWithTypeAndName arg1 = arguments[0]; - const auto * src_col = checkAndGetColumn(arg1.column.get()); - size_t size = src_col->size(); - - if (!result_type->isNullable()) - throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s return type must be nullable", name); - - if (!isDate32(removeNullable(result_type))) - throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s return type must be date32.", name); - - using ColVecTo = DB::DataTypeDate32::ColumnType; - typename ColVecTo::MutablePtr result_column = ColVecTo::create(size, 0); - typename ColVecTo::Container & result_container = result_column->getData(); - DB::ColumnUInt8::MutablePtr null_map = DB::ColumnUInt8::create(size, 0); - typename DB::ColumnUInt8::Container & null_container = null_map->getData(); - const DateLUTImpl * local_time_zone = &DateLUT::instance(); - const DateLUTImpl * utc_time_zone = &DateLUT::instance("UTC"); - - for (size_t i = 0; i < size; ++i) - { - auto str = src_col->getDataAt(i); - if (str.size < 4) - { - null_container[i] = true; - continue; - } - else - { - DB::ReadBufferFromMemory buf(str.data, str.size); - while(!buf.eof() && *buf.position() == ' ') - { - buf.position() ++; - } - if(buf.buffer().end() - buf.position() < 4) - { - null_container[i] = true; - continue; - } - UInt8 can_be_parsed = 1; - if (!checkAndGetDate32(buf, result_container[i], *local_time_zone, can_be_parsed)) - { - if (!can_be_parsed) - null_container[i] = true; - else - { - time_t tmp = 0; - bool parsed = tryParseDateTimeBestEffort(tmp, buf, *local_time_zone, *utc_time_zone); - result_container[i] = local_time_zone->toDayNum(tmp); - null_container[i] = !parsed; - } - } - } - } - return DB::ColumnNullable::create(std::move(result_column), std::move(null_map)); + auto data_type = std::make_shared(); + return makeNullable(data_type); } }; diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index aa7b95d6782d..543489c2e08f 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -902,6 +902,22 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( return &actions_dag->addAlias(actions_dag->findInOutputs(result_name), result_name); } + if (ch_func_name == "toYear") + { + const ActionsDAG::Node * arg_node = args[0]; + const String & arg_func_name = arg_node->function ? arg_node->function->getName() : ""; + if ((arg_func_name == "sparkToDate" || arg_func_name == "sparkToDateTime") && arg_node->children.size() > 0) + { + const ActionsDAG::Node * child_node = arg_node->children[0]; + if (child_node && isString(removeNullable(child_node->result_type))) + { + auto extract_year_builder = FunctionFactory::instance().get("sparkExtractYear", context); + auto func_result_name = "sparkExtractYear(" + child_node->result_name + ")"; + return &actions_dag->addFunction(extract_year_builder, {child_node}, func_result_name); + } + } + } + const ActionsDAG::Node * result_node; if (ch_func_name == "splitByRegexp") From c9018cdd884c5e93a171ba1561888e719026898c Mon Sep 17 00:00:00 2001 From: Chang chen Date: Wed, 8 May 2024 13:55:59 +0800 Subject: [PATCH 023/402] [GLUTEN-5620][CORE] Simplify Decimal process logic (#5621) * rescaleCastForDecimal refactor * refactor isPromoteCast * Simplify Decimal process logic and re-implement FunctionParserDivide, so divide.cpp is deleted. * remove SerializedPlanParser::convertBinaryArithmeticFunDecimalArgs * rename noCheckOverflow to dontTransformCheckOverflow * update per comments * fix warning * fix style warning * fix typo --- .../backendsapi/clickhouse/CHBackend.scala | 1 + cpp-ch/local-engine/Common/CHUtil.cpp | 63 ++- cpp-ch/local-engine/Common/CHUtil.h | 16 +- .../Parser/SerializedPlanParser.cpp | 82 +--- .../Parser/SerializedPlanParser.h | 12 +- .../scalar_function_parser/arithmetic.cpp | 399 ++++++++++++++++++ .../Parser/scalar_function_parser/divide.cpp | 68 --- .../backendsapi/BackendSettingsApi.scala | 8 + .../expression/ExpressionConverter.scala | 134 +++--- .../gluten/utils/DecimalArithmeticUtil.scala | 83 ++-- 10 files changed, 593 insertions(+), 273 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp delete mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/divide.cpp diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index 276ce11fb4dc..da6c60d8aea1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -257,6 +257,7 @@ object CHBackendSettings extends BackendSettingsApi with Logging { override def needOutputSchemaForPlan(): Boolean = true override def allowDecimalArithmetic: Boolean = !SQLConf.get.decimalOperationsAllowPrecisionLoss + override def transformCheckOverflow: Boolean = false override def requiredInputFilePaths(): Boolean = true diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 9704b3041cd9..9e2ce6304718 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include "CHUtil.h" #include #include #include +#include #include #include #include #include -#include #include #include #include @@ -30,14 +32,17 @@ #include #include #include -#include +#include #include #include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -51,8 +56,11 @@ #include #include #include +#include +#include #include #include +#include #include #include #include @@ -63,20 +71,12 @@ #include #include -#include -#include - -#include "CHUtil.h" -#include "Disks/registerGlutenDisks.h" - -#include -#include - namespace DB { namespace ErrorCodes { extern const int BAD_ARGUMENTS; +extern const int UNKNOWN_TYPE; } } @@ -311,16 +311,48 @@ size_t PODArrayUtil::adjustMemoryEfficientSize(size_t n) std::string PlanUtil::explainPlan(DB::QueryPlan & plan) { - std::string plan_str; - DB::QueryPlan::ExplainPlanOptions buf_opt{ + constexpr DB::QueryPlan::ExplainPlanOptions buf_opt{ .header = true, .actions = true, .indexes = true, }; DB::WriteBufferFromOwnString buf; plan.explainPlan(buf, buf_opt); - plan_str = buf.str(); - return plan_str; + + return buf.str(); +} + +void PlanUtil::checkOuputType(const DB::QueryPlan & plan) +{ + // QueryPlan::checkInitialized is a private method, so we assume plan is initialized, otherwise there is a core dump here. + // It's okay, because it's impossible for us not to initialize where we call this method. + const auto & step = *plan.getRootNode()->step; + if (!step.hasOutputStream()) + return; + if (!step.getOutputStream().header) + return; + for (const auto & elem : step.getOutputStream().header) + { + const DB::DataTypePtr & ch_type = elem.type; + const auto ch_type_without_nullable = DB::removeNullable(ch_type); + const DB::WhichDataType which(ch_type_without_nullable); + if (which.isDateTime64()) + { + const auto * ch_type_datetime64 = checkAndGetDataType(ch_type_without_nullable.get()); + if (ch_type_datetime64->getScale() != 6) + throw Exception(ErrorCodes::UNKNOWN_TYPE, "Spark doesn't support converting from {}", ch_type->getName()); + } + else if (which.isDecimal()) + { + if (which.isDecimal256()) + throw Exception(ErrorCodes::UNKNOWN_TYPE, "Spark doesn't support converting from {}", ch_type->getName()); + + const auto scale = getDecimalScale(*ch_type_without_nullable); + const auto precision = getDecimalPrecision(*ch_type_without_nullable); + if (scale == 0 && precision == 0) + throw Exception(ErrorCodes::UNKNOWN_TYPE, "Spark doesn't support converting from {}", ch_type->getName()); + } + } } NestedColumnExtractHelper::NestedColumnExtractHelper(const DB::Block & block_, bool case_insentive_) @@ -713,7 +745,6 @@ void registerAllFunctions() auto & factory = AggregateFunctionCombinatorFactory::instance(); registerAggregateFunctionCombinatorPartialMerge(factory); } - } void registerGlutenDisks() diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 574cdbe4c8d7..edbd91c50d22 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -16,19 +16,21 @@ */ #pragma once #include -#include #include #include #include -#include #include #include #include #include -#include #include #include -#include + +namespace DB +{ +class QueryPipeline; +class QueryPlan; +} namespace local_engine { @@ -96,10 +98,10 @@ class NestedColumnExtractHelper const DB::ColumnWithTypeAndName * findColumn(const DB::Block & block, const std::string & name) const; }; -class PlanUtil +namespace PlanUtil { -public: - static std::string explainPlan(DB::QueryPlan & plan); +std::string explainPlan(DB::QueryPlan & plan); +void checkOuputType(const DB::QueryPlan & plan); }; class ActionsDAGUtil diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 543489c2e08f..82acba37f7d8 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -867,8 +867,7 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( auto pos = function_signature.find(':'); auto func_name = function_signature.substr(0, pos); - auto func_parser = FunctionParserFactory::instance().tryGet(func_name, this); - if (func_parser) + if (auto func_parser = FunctionParserFactory::instance().tryGet(func_name, this)) { LOG_DEBUG( &Poco::Logger::get("SerializedPlanParser"), @@ -971,13 +970,12 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( args = std::move(new_args); } - bool converted_decimal_args = convertBinaryArithmeticFunDecimalArgs(actions_dag, args, scalar_function); auto function_builder = FunctionFactory::instance().get(ch_func_name, context); std::string args_name = join(args, ','); result_name = ch_func_name + "(" + args_name + ")"; const auto * function_node = &actions_dag->addFunction(function_builder, args, result_name); result_node = function_node; - if (!TypeParser::isTypeMatched(rel.scalar_function().output_type(), function_node->result_type) && !converted_decimal_args) + if (!TypeParser::isTypeMatched(rel.scalar_function().output_type(), function_node->result_type)) { auto result_type = TypeParser::parseType(rel.scalar_function().output_type()); if (isDecimalOrNullableDecimal(result_type)) @@ -1014,76 +1012,6 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( return result_node; } -bool SerializedPlanParser::convertBinaryArithmeticFunDecimalArgs( - ActionsDAGPtr actions_dag, - ActionsDAG::NodeRawConstPtrs & args, - const substrait::Expression_ScalarFunction & arithmeticFun) -{ - auto function_signature = function_mapping.at(std::to_string(arithmeticFun.function_reference())); - auto pos = function_signature.find(':'); - auto func_name = function_signature.substr(0, pos); - - if (func_name == "divide" || func_name == "multiply" || func_name == "plus" || func_name == "minus") - { - /// for divide/plus/minus, we need to convert first arg to result precision and scale - /// for multiply, we need to convert first arg to result precision, but keep scale - auto arg1_type = removeNullable(args[0]->result_type); - auto arg2_type = removeNullable(args[1]->result_type); - if (isDecimal(arg1_type) && isDecimal(arg2_type)) - { - UInt32 p1 = getDecimalPrecision(*arg1_type); - UInt32 s1 = getDecimalScale(*arg1_type); - UInt32 p2 = getDecimalPrecision(*arg2_type); - UInt32 s2 = getDecimalScale(*arg2_type); - - UInt32 precision; - UInt32 scale; - - if (func_name == "plus" || func_name == "minus") - { - scale = s1; - precision = scale + std::max(p1 - s1, p2 - s2) + 1; - } - else if (func_name == "divide") - { - scale = std::max(static_cast(6), s1 + p2 + 1); - precision = p1 - s1 + s2 + scale; - } - else // multiply - { - scale = s1; - precision = p1 + p2 + 1; - } - - UInt32 maxPrecision = DataTypeDecimal256::maxPrecision(); - UInt32 maxScale = DataTypeDecimal128::maxPrecision(); - precision = std::min(precision, maxPrecision); - scale = std::min(scale, maxScale); - - ActionsDAG::NodeRawConstPtrs new_args; - new_args.reserve(args.size()); - - ActionsDAG::NodeRawConstPtrs cast_args; - cast_args.reserve(2); - cast_args.emplace_back(args[0]); - DataTypePtr ch_type = createDecimal(precision, scale); - ch_type = wrapNullableType(arithmeticFun.output_type().decimal().nullability(), ch_type); - String type_name = ch_type->getName(); - DataTypePtr str_type = std::make_shared(); - const ActionsDAG::Node * type_node = &actions_dag->addColumn( - ColumnWithTypeAndName(str_type->createColumnConst(1, type_name), str_type, getUniqueName(type_name))); - cast_args.emplace_back(type_node); - const ActionsDAG::Node * cast_node = toFunctionNode(actions_dag, "CAST", cast_args); - actions_dag->addOrReplaceInOutputs(*cast_node); - new_args.emplace_back(cast_node); - new_args.emplace_back(args[1]); - args = std::move(new_args); - return true; - } - } - return false; -} - void SerializedPlanParser::parseFunctionArguments( ActionsDAGPtr & actions_dag, ActionsDAG::NodeRawConstPtrs & parsed_args, @@ -1835,11 +1763,15 @@ QueryPlanPtr SerializedPlanParser::parse(const std::string & plan) auto res = parse(std::move(plan_ptr)); +#ifndef NDEBUG + PlanUtil::checkOuputType(*res); +#endif + auto * logger = &Poco::Logger::get("SerializedPlanParser"); if (logger->debug()) { auto out = PlanUtil::explainPlan(*res); - LOG_DEBUG(logger, "clickhouse plan:\n{}", out); + LOG_ERROR(logger, "clickhouse plan:\n{}", out); } return res; } diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 5bf7da25d32c..a636ebb9352f 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -17,7 +17,6 @@ #pragma once #include -#include #include #include #include @@ -25,14 +24,10 @@ #include #include #include -#include #include #include -#include #include -#include #include -#include #include #include #include @@ -301,9 +296,6 @@ class SerializedPlanParser static std::string getFunctionName(const std::string & function_sig, const substrait::Expression_ScalarFunction & function); - bool convertBinaryArithmeticFunDecimalArgs( - ActionsDAGPtr actions_dag, ActionsDAG::NodeRawConstPtrs & args, const substrait::Expression_ScalarFunction & arithmeticFun); - IQueryPlanStep * addRemoveNullableStep(QueryPlan & plan, const std::set & columns); static ContextMutablePtr global_context; @@ -383,7 +375,6 @@ class SerializedPlanParser void wrapNullable( const std::vector & columns, ActionsDAGPtr actions_dag, std::map & nullable_measure_names); static std::pair convertStructFieldType(const DB::DataTypePtr & type, const DB::Field & field); - const ActionsDAG::Node * addColumn(DB::ActionsDAGPtr actions_dag, const DataTypePtr & type, const Field & field); int name_no = 0; std::unordered_map function_mapping; @@ -395,6 +386,9 @@ class SerializedPlanParser // for parse rel node, collect steps from a rel node std::vector temp_step_collection; std::vector metrics; + +public: + const ActionsDAG::Node * addColumn(DB::ActionsDAGPtr actions_dag, const DataTypePtr & type, const Field & field); }; struct SparkBuffer diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp new file mode 100644 index 000000000000..ec056da45e07 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +namespace DB::ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +namespace local_engine +{ + +class DecimalType +{ + static constexpr Int32 spark_max_precision = 38; + static constexpr Int32 spark_max_scale = 38; + static constexpr Int32 minimum_adjusted_scale = 6; + + static constexpr Int32 chickhouse_max_precision = DB::DataTypeDecimal256::maxPrecision(); + static constexpr Int32 chickhouse_max_scale = DB::DataTypeDecimal128::maxPrecision(); + +public: + Int32 precision; + Int32 scale; + +private: + static DecimalType bounded_to_spark(const Int32 precision, const Int32 scale) + { + return DecimalType(std::min(precision, spark_max_precision), std::min(scale, spark_max_scale)); + } + static DecimalType bounded_to_click_house(const Int32 precision, const Int32 scale) + { + return DecimalType(std::min(precision, chickhouse_max_precision), std::min(scale, chickhouse_max_scale)); + } + static void check_negative_scale(const Int32 scale) + { + /// only support spark.sql.legacy.allowNegativeScaleOfDecimal == false + if (scale < 0) + throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Negative scale is not supported"); + } + + static DecimalType adjust_precision_scale(const Int32 precision, const Int32 scale) + { + check_negative_scale(scale); + assert(precision >= scale); + + if (precision <= spark_max_precision) + { + // Adjustment only needed when we exceed max precision + return DecimalType(precision, scale); + } + else if (scale < 0) + { + // Decimal can have negative scale (SPARK-24468). In this case, we cannot allow a precision + // loss since we would cause a loss of digits in the integer part. + // In this case, we are likely to meet an overflow. + return DecimalType(spark_max_precision, scale); + } + else + { + // Precision/scale exceed maximum precision. Result must be adjusted to MAX_PRECISION. + const int intDigits = precision - scale; + + // If original scale is less than MINIMUM_ADJUSTED_SCALE, use original scale value; otherwise + // preserve at least MINIMUM_ADJUSTED_SCALE fractional digits + const int minScaleValue = std::min(scale, minimum_adjusted_scale); + + // The resulting scale is the maximum between what is available without causing a loss of + // digits for the integer part of the decimal and the minimum guaranteed scale, which is + // computed above + const int adjusted_scale = std::max(spark_max_precision - intDigits, minScaleValue); + return DecimalType(spark_max_precision, adjusted_scale); + } + } + +public: + /// The formula follows Hive which is based on the SQL standard and MS SQL: + /// https://cwiki.apache.org/confluence/download/attachments/27362075/Hive_Decimal_Precision_Scale_Support.pdf + /// https://msdn.microsoft.com/en-us/library/ms190476.aspx + /// Result Precision: max(s1, s2) + max(p1-s1, p2-s2) + 1 + /// Result Scale: max(s1, s2) + /// +, - + static DecimalType + resultAddSubstractDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2, bool allowPrecisionLoss = true) + { + const Int32 scale = std::max(s1, s2); + const Int32 precision = std::max(p1 - s1, p2 - s2) + scale + 1; + + if (allowPrecisionLoss) + return adjust_precision_scale(precision, scale); + else + return bounded_to_spark(precision, scale); + } + + /// The formula follows Hive which is based on the SQL standard and MS SQL: + /// https://cwiki.apache.org/confluence/download/attachments/27362075/Hive_Decimal_Precision_Scale_Support.pdf + /// https://msdn.microsoft.com/en-us/library/ms190476.aspx + /// Result Precision: p1 - s1 + s2 + max(6, s1 + p2 + 1) + /// Result Scale: max(6, s1 + p2 + 1) + static DecimalType + resultDivideDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2, bool allowPrecisionLoss = true) + { + if (allowPrecisionLoss) + { + const Int32 Int32Dig = p1 - s1 + s2; + const Int32 scale = std::max(minimum_adjusted_scale, s1 + p2 + 1); + const Int32 prec = Int32Dig + scale; + return adjust_precision_scale(prec, scale); + } + else + { + Int32 Int32Dig = std::min(spark_max_scale, p1 - s1 + s2); + Int32 decDig = std::min(spark_max_scale, std::max(minimum_adjusted_scale, s1 + p2 + 1)); + Int32 diff = (Int32Dig + decDig) - spark_max_scale; + + if (diff > 0) + { + decDig -= diff / 2 + 1; + Int32Dig = spark_max_scale - decDig; + } + + return bounded_to_spark(Int32Dig + decDig, decDig); + } + } + + /// The formula follows Hive which is based on the SQL standard and MS SQL: + /// https://cwiki.apache.org/confluence/download/attachments/27362075/Hive_Decimal_Precision_Scale_Support.pdf + /// https://msdn.microsoft.com/en-us/library/ms190476.aspx + /// Result Precision: p1 + p2 + 1 + /// Result Scale: s1 + s2 + static DecimalType + resultMultiplyDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2, bool allowPrecisionLoss = true) + { + const Int32 scale = s1 + s2; + const Int32 precision = p1 + p2 + 1; + + if (allowPrecisionLoss) + return adjust_precision_scale(precision, scale); + else + return bounded_to_spark(precision, scale); + } + + static DecimalType evalAddSubstractDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) + { + const Int32 scale = s1; + const Int32 precision = scale + std::max(p1 - s1, p2 - s2) + 1; + return bounded_to_click_house(precision, scale); + } + + static DecimalType evalDividetDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) + { + const Int32 scale = std::max(minimum_adjusted_scale, s1 + p2 + 1); + const Int32 precision = p1 - s1 + s2 + scale; + return bounded_to_click_house(precision, scale); + } + + static DecimalType evalMultiplyDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) + { + const Int32 scale = s1; + const Int32 precision = p1 + p2 + 1; + return bounded_to_click_house(precision, scale); + } +}; + +class FunctionParserBinaryArithmetic : public FunctionParser +{ +protected: + ActionsDAG::NodeRawConstPtrs convertBinaryArithmeticFunDecimalArgs( + ActionsDAGPtr & actions_dag, + const ActionsDAG::NodeRawConstPtrs & args, + const DecimalType & eval_type, + const substrait::Expression_ScalarFunction & arithmeticFun) const + { + const Int32 precision = eval_type.precision; + const Int32 scale = eval_type.scale; + + ActionsDAG::NodeRawConstPtrs new_args; + new_args.reserve(args.size()); + + ActionsDAG::NodeRawConstPtrs cast_args; + cast_args.reserve(2); + cast_args.emplace_back(args[0]); + DataTypePtr ch_type = createDecimal(precision, scale); + ch_type = wrapNullableType(arithmeticFun.output_type().decimal().nullability(), ch_type); + const String type_name = ch_type->getName(); + const DataTypePtr str_type = std::make_shared(); + const ActionsDAG::Node * type_node + = &actions_dag->addColumn(ColumnWithTypeAndName(str_type->createColumnConst(1, type_name), str_type, getUniqueName(type_name))); + cast_args.emplace_back(type_node); + const ActionsDAG::Node * cast_node = toFunctionNode(actions_dag, "CAST", cast_args); + actions_dag->addOrReplaceInOutputs(*cast_node); + new_args.emplace_back(cast_node); + new_args.emplace_back(args[1]); + return new_args; + } + + DecimalType getDecimalType(const DataTypePtr & left, const DataTypePtr & right, const bool resultType) const + { + assert(isDecimal(left) && isDecimal(right)); + const Int32 p1 = getDecimalPrecision(*left); + const Int32 s1 = getDecimalScale(*left); + const Int32 p2 = getDecimalPrecision(*right); + const Int32 s2 = getDecimalScale(*right); + return resultType ? internalResultType(p1, s1, p2, s2) : internalEvalType(p1, s1, p2, s2); + } + + virtual DecimalType internalResultType(Int32 p1, Int32 s1, Int32 p2, Int32 s2) const = 0; + virtual DecimalType internalEvalType(Int32 p1, Int32 s1, Int32 p2, Int32 s2) const = 0; + + const ActionsDAG::Node * + checkDecimalOverflow(ActionsDAGPtr & actions_dag, const ActionsDAG::Node * func_node, Int32 precision, Int32 scale) const + { + const DB::ActionsDAG::NodeRawConstPtrs overflow_args + = {func_node, + plan_parser->addColumn(actions_dag, std::make_shared(), precision), + plan_parser->addColumn(actions_dag, std::make_shared(), scale)}; + return toFunctionNode(actions_dag, "checkDecimalOverflowSparkOrNull", overflow_args); + } + const DB::ActionsDAG::Node * convertNodeTypeIfNeeded( + const substrait::Expression_ScalarFunction & substrait_func, + const DB::ActionsDAG::Node * func_node, + DB::ActionsDAGPtr & actions_dag) const override + { + const auto & substrait_type = substrait_func.output_type(); + if (const auto result_type = TypeParser::parseType(substrait_type); isDecimalOrNullableDecimal(result_type)) + { + const auto a = removeNullable(result_type); + const auto b = removeNullable(func_node->result_type); + if (a->equals(*b)) + return func_node; + + // as stated in isTypeMatched， currently we don't change nullability of the result type + const std::string type_name = func_node->result_type->isNullable() ? wrapNullableType(true, result_type)->getName() + : removeNullable(result_type)->getName(); + return ActionsDAGUtil::convertNodeType(actions_dag, func_node, type_name, func_node->result_name, DB::CastType::accurateOrNull); + } + return FunctionParser::convertNodeTypeIfNeeded(substrait_func, func_node, actions_dag); + } + + virtual const DB::ActionsDAG::Node * + createFunctionNode(DB::ActionsDAGPtr & actions_dag, const String & func_name, const DB::ActionsDAG::NodeRawConstPtrs & args) const + { + return toFunctionNode(actions_dag, func_name, args); + } + +public: + explicit FunctionParserBinaryArithmetic(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } + const ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const override + { + const auto ch_func_name = getCHFunctionName(substrait_func); + auto parsed_args = parseFunctionArguments(substrait_func, ch_func_name, actions_dag); + + if (parsed_args.size() != 2) + throw Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly two arguments", getName()); + + const auto left_type = DB::removeNullable(parsed_args[0]->result_type); + const auto right_type = DB::removeNullable(parsed_args[1]->result_type); + const bool converted = isDecimal(left_type) && isDecimal(right_type); + + if (converted) + { + const DecimalType evalType = getDecimalType(left_type, right_type, false); + parsed_args = convertBinaryArithmeticFunDecimalArgs(actions_dag, parsed_args, evalType, substrait_func); + } + + const auto * func_node = createFunctionNode(actions_dag, ch_func_name, parsed_args); + + if (converted) + { + const auto parsed_outputType = removeNullable(TypeParser::parseType(substrait_func.output_type())); + assert(isDecimal(parsed_outputType)); + const Int32 parsed_precision = getDecimalPrecision(*parsed_outputType); + const Int32 parsed_scale = getDecimalScale(*parsed_outputType); + +#ifndef NDEBUG + const auto [precision, scale] = getDecimalType(left_type, right_type, true); + // assert(parsed_precision == precision); + // assert(parsed_scale == scale); +#endif + func_node = checkDecimalOverflow(actions_dag, func_node, parsed_precision, parsed_scale); + } + return convertNodeTypeIfNeeded(substrait_func, func_node, actions_dag); + } +}; + +class FunctionParserPlus final : public FunctionParserBinaryArithmetic +{ +public: + explicit FunctionParserPlus(SerializedPlanParser * plan_parser_) : FunctionParserBinaryArithmetic(plan_parser_) { } + + static constexpr auto name = "add"; + String getName() const override { return name; } + +protected: + DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::resultAddSubstractDecimalType(p1, s1, p2, s2); + } + DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::evalAddSubstractDecimalType(p1, s1, p2, s2); + } +}; + +class FunctionParserMinus final : public FunctionParserBinaryArithmetic +{ +public: + explicit FunctionParserMinus(SerializedPlanParser * plan_parser_) : FunctionParserBinaryArithmetic(plan_parser_) { } + + static constexpr auto name = "subtract"; + String getName() const override { return name; } + +protected: + DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::resultAddSubstractDecimalType(p1, s1, p2, s2); + } + DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::evalAddSubstractDecimalType(p1, s1, p2, s2); + } +}; + +class FunctionParserMultiply final : public FunctionParserBinaryArithmetic +{ +public: + explicit FunctionParserMultiply(SerializedPlanParser * plan_parser_) : FunctionParserBinaryArithmetic(plan_parser_) { } + static constexpr auto name = "multiply"; + String getName() const override { return name; } + +protected: + DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::resultMultiplyDecimalType(p1, s1, p2, s2); + } + DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::evalMultiplyDecimalType(p1, s1, p2, s2); + } +}; + +class FunctionParserDivide final : public FunctionParserBinaryArithmetic +{ +public: + explicit FunctionParserDivide(SerializedPlanParser * plan_parser_) : FunctionParserBinaryArithmetic(plan_parser_) { } + static constexpr auto name = "divide"; + String getName() const override { return name; } + +protected: + DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::resultDivideDecimalType(p1, s1, p2, s2); + } + DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::evalDividetDecimalType(p1, s1, p2, s2); + } + + const DB::ActionsDAG::Node * createFunctionNode( + DB::ActionsDAGPtr & actions_dag, const String & func_name, const DB::ActionsDAG::NodeRawConstPtrs & new_args) const override + { + assert(func_name == name); + const auto * left_arg = new_args[0]; + const auto * right_arg = new_args[1]; + + if (isDecimal(removeNullable(left_arg->result_type)) || isDecimal(removeNullable(right_arg->result_type))) + return toFunctionNode(actions_dag, "sparkDivideDecimal", {left_arg, right_arg}); + else + return toFunctionNode(actions_dag, "sparkDivide", {left_arg, right_arg}); + } +}; + +static FunctionParserRegister register_plus; +static FunctionParserRegister register_minus; +static FunctionParserRegister register_mltiply; +static FunctionParserRegister register_divide; + +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/divide.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/divide.cpp deleted file mode 100644 index 5c1eb358b31d..000000000000 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/divide.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} -} - -namespace local_engine -{ - -class FunctionParserDivide : public FunctionParser -{ -public: - explicit FunctionParserDivide(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } - ~FunctionParserDivide() override = default; - - static constexpr auto name = "divide"; - - String getName() const override { return name; } - - const ActionsDAG::Node * parse( - const substrait::Expression_ScalarFunction & substrait_func, - ActionsDAGPtr & actions_dag) const override - { - /// Parse divide(left, right) as if (right == 0) null else left / right - auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); - if (parsed_args.size() != 2) - throw Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly two arguments", getName()); - - ActionsDAG::NodeRawConstPtrs new_args{parsed_args[0], parsed_args[1]}; - plan_parser->convertBinaryArithmeticFunDecimalArgs(actions_dag, new_args, substrait_func); - - const auto * left_arg = new_args[0]; - const auto * right_arg = new_args[1]; - - if (isDecimal(removeNullable(left_arg->result_type)) || isDecimal(removeNullable(right_arg->result_type))) - return toFunctionNode(actions_dag, "sparkDivideDecimal", {left_arg, right_arg}); - else - return toFunctionNode(actions_dag, "sparkDivide", {left_arg, right_arg}); - } -}; - -static FunctionParserRegister register_divide; -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index fa795b84b64d..ddf62201daeb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -98,6 +98,14 @@ trait BackendSettingsApi { def allowDecimalArithmetic: Boolean = true + /** + * After https://github.com/apache/spark/pull/36698, every arithmetic should report the accurate + * result decimal type and implement `CheckOverflow` by itself.

Regardless of whether there + * is 36698 or not, this option is used to indicate whether to transform `CheckOverflow`. `false` + * means the backend will implement `CheckOverflow` by default and no need to transform it. + */ + def transformCheckOverflow: Boolean = true + def rescaleDecimalIntegralExpression(): Boolean = false def shuffleSupportedCodec(): Set[String] diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 7815cbf69ebd..562ae294e2c0 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -119,7 +119,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { case i: StaticInvoke => val objectName = i.staticObject.getName.stripSuffix("$") if (objectName.endsWith("UrlCodec")) { - val child = i.arguments(0) + val child = i.arguments.head i.functionName match { case "decode" => return GenericExpressionTransformer( @@ -138,20 +138,8 @@ object ExpressionConverter extends SQLConfHelper with Logging { case _ => } - TestStats.addExpressionClassName(expr.getClass.getName) - // Check whether Gluten supports this expression - val substraitExprNameOpt = expressionsMap.get(expr.getClass) - if (substraitExprNameOpt.isEmpty) { - throw new GlutenNotSupportException( - s"Not supported to map spark function name" + - s" to substrait function name: $expr, class name: ${expr.getClass.getSimpleName}.") - } - val substraitExprName = substraitExprNameOpt.get + val substraitExprName: String = getAndCheckSubstraitName(expr, expressionsMap) - // Check whether each backend supports this expression - if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(substraitExprName, expr)) { - throw new GlutenNotSupportException(s"Not supported: $expr.") - } expr match { case extendedExpr if ExpressionMappings.expressionExtensionTransformer.extensionExpressionsMapping.contains( @@ -162,7 +150,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { case c: CreateArray => val children = c.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) - CreateArrayTransformer(substraitExprName, children, true, c) + CreateArrayTransformer(substraitExprName, children, useStringTypeWhenEmpty = true, c) case g: GetArrayItem => GetArrayItemTransformer( substraitExprName, @@ -319,7 +307,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { i.hset, i.child.dataType, i) - case s: org.apache.spark.sql.execution.ScalarSubquery => + case s: ScalarSubquery => ScalarSubqueryTransformer(s.plan, s.exprId, s) case c: Cast => // Add trim node, as necessary. @@ -463,7 +451,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { expressionsMap), arguments = lambdaFunction.arguments.map( replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), - hidden = false, original = lambdaFunction ) case j: JsonTuple => @@ -477,11 +464,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(l.right, attributeSeq, expressionsMap), l ) - case c: CheckOverflow => - CheckOverflowTransformer( - substraitExprName, - replaceWithExpressionTransformerInternal(c.child, attributeSeq, expressionsMap), - c) case m: MakeDecimal => MakeDecimalTransformer( substraitExprName, @@ -510,42 +492,71 @@ object ExpressionConverter extends SQLConfHelper with Logging { expr.children.map( replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), expr) - case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b) => - // PrecisionLoss=true: velox support / ch not support - // PrecisionLoss=false: velox not support / ch support - // TODO ch support PrecisionLoss=true - if (!BackendsApiManager.getSettings.allowDecimalArithmetic) { - throw new GlutenNotSupportException( - s"Not support ${SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key} " + - s"${conf.decimalOperationsAllowPrecisionLoss} mode") - } - val rescaleBinary = if (BackendsApiManager.getSettings.rescaleDecimalLiteral) { - DecimalArithmeticUtil.rescaleLiteral(b) - } else { - b - } - val (left, right) = DecimalArithmeticUtil.rescaleCastForDecimal( - DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.left), - DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.right)) - val leftChild = replaceWithExpressionTransformerInternal(left, attributeSeq, expressionsMap) - val rightChild = - replaceWithExpressionTransformerInternal(right, attributeSeq, expressionsMap) - val resultType = DecimalArithmeticUtil.getResultTypeForOperation( - DecimalArithmeticUtil.getOperationType(b), - DecimalArithmeticUtil - .getResultType(leftChild) - .getOrElse(left.dataType.asInstanceOf[DecimalType]), - DecimalArithmeticUtil - .getResultType(rightChild) - .getOrElse(right.dataType.asInstanceOf[DecimalType]) - ) + case CheckOverflow(b: BinaryArithmetic, decimalType, _) + if !BackendsApiManager.getSettings.transformCheckOverflow && + DecimalArithmeticUtil.isDecimalArithmetic(b) => + DecimalArithmeticUtil.checkAllowDecimalArithmetic() + val leftChild = + replaceWithExpressionTransformerInternal(b.left, attributeSeq, expressionsMap) + val rightChild = + replaceWithExpressionTransformerInternal(b.right, attributeSeq, expressionsMap) DecimalArithmeticExpressionTransformer( - substraitExprName, + getAndCheckSubstraitName(b, expressionsMap), leftChild, rightChild, - resultType, + decimalType, b) + + case c: CheckOverflow => + CheckOverflowTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(c.child, attributeSeq, expressionsMap), + c) + + case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b) => + DecimalArithmeticUtil.checkAllowDecimalArithmetic() + if (!BackendsApiManager.getSettings.transformCheckOverflow) { + val leftChild = + replaceWithExpressionTransformerInternal(b.left, attributeSeq, expressionsMap) + val rightChild = + replaceWithExpressionTransformerInternal(b.right, attributeSeq, expressionsMap) + DecimalArithmeticExpressionTransformer( + substraitExprName, + leftChild, + rightChild, + b.dataType.asInstanceOf[DecimalType], + b) + } else { + val rescaleBinary = if (BackendsApiManager.getSettings.rescaleDecimalLiteral) { + DecimalArithmeticUtil.rescaleLiteral(b) + } else { + b + } + val (left, right) = DecimalArithmeticUtil.rescaleCastForDecimal( + DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.left), + DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.right)) + val leftChild = + replaceWithExpressionTransformerInternal(left, attributeSeq, expressionsMap) + val rightChild = + replaceWithExpressionTransformerInternal(right, attributeSeq, expressionsMap) + + val resultType = DecimalArithmeticUtil.getResultTypeForOperation( + DecimalArithmeticUtil.getOperationType(b), + DecimalArithmeticUtil + .getResultType(leftChild) + .getOrElse(left.dataType.asInstanceOf[DecimalType]), + DecimalArithmeticUtil + .getResultType(rightChild) + .getOrElse(right.dataType.asInstanceOf[DecimalType]) + ) + DecimalArithmeticExpressionTransformer( + substraitExprName, + leftChild, + rightChild, + resultType, + b) + } case n: NaNvl => BackendsApiManager.getSparkPlanExecApiInstance.genNaNvlTransformer( substraitExprName, @@ -651,6 +662,23 @@ object ExpressionConverter extends SQLConfHelper with Logging { } } + private def getAndCheckSubstraitName(expr: Expression, expressionsMap: Map[Class[_], String]) = { + TestStats.addExpressionClassName(expr.getClass.getName) + // Check whether Gluten supports this expression + val substraitExprNameOpt = expressionsMap.get(expr.getClass) + if (substraitExprNameOpt.isEmpty) { + throw new GlutenNotSupportException( + s"Not supported to map spark function name" + + s" to substrait function name: $expr, class name: ${expr.getClass.getSimpleName}.") + } + val substraitExprName = substraitExprNameOpt.get + // Check whether each backend supports this expression + if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(substraitExprName, expr)) { + throw new GlutenNotSupportException(s"Not supported: $expr.") + } + substraitExprName + } + /** * Transform BroadcastExchangeExec to ColumnarBroadcastExchangeExec in DynamicPruningExpression. * diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala index 621dcc061ec7..ff63a1726393 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala @@ -19,11 +19,15 @@ package org.apache.gluten.utils import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression.{CheckOverflowTransformer, ChildTransformer, DecimalArithmeticExpressionTransformer, ExpressionTransformer} +import org.apache.gluten.expression.ExpressionConverter.conf import org.apache.spark.sql.catalyst.analysis.DecimalPrecision import org.apache.spark.sql.catalyst.expressions.{Add, BinaryArithmetic, Cast, Divide, Expression, Literal, Multiply, Pmod, PromotePrecision, Remainder, Subtract} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ByteType, Decimal, DecimalType, IntegerType, LongType, ShortType} +import scala.annotation.tailrec + object DecimalArithmeticUtil { object OperationType extends Enumeration { @@ -31,7 +35,7 @@ object DecimalArithmeticUtil { val ADD, SUBTRACT, MULTIPLY, DIVIDE, MOD = Value } - val MIN_ADJUSTED_SCALE = 6 + private val MIN_ADJUSTED_SCALE = 6 val MAX_PRECISION = 38 // Returns the result decimal type of a decimal arithmetic computing. @@ -67,7 +71,7 @@ object DecimalArithmeticUtil { } // Returns the adjusted decimal type when the precision is larger the maximum. - def adjustScaleIfNeeded(precision: Int, scale: Int): DecimalType = { + private def adjustScaleIfNeeded(precision: Int, scale: Int): DecimalType = { var typePrecision = precision var typeScale = scale if (precision > MAX_PRECISION) { @@ -159,56 +163,33 @@ object DecimalArithmeticUtil { } // Returns whether the input expression is a combination of PromotePrecision(Cast as DecimalType). - private def isPromoteCast(expr: Expression): Boolean = { - expr match { - case precision: PromotePrecision => - precision.child match { - case cast: Cast if cast.dataType.isInstanceOf[DecimalType] => true - case _ => false - } - case _ => false - } + private def isPromoteCast(expr: Expression): Boolean = expr match { + case PromotePrecision(Cast(_, _: DecimalType, _, _)) => true + case _ => false } def rescaleCastForDecimal(left: Expression, right: Expression): (Expression, Expression) = { - if (!BackendsApiManager.getSettings.rescaleDecimalIntegralExpression()) { - return (left, right) + + def doScale(e1: Expression, e2: Expression): (Expression, Expression) = { + val newE2 = rescaleCastForOneSide(e2) + val isWiderType = checkIsWiderType( + e1.dataType.asInstanceOf[DecimalType], + newE2.dataType.asInstanceOf[DecimalType], + e2.dataType.asInstanceOf[DecimalType]) + if (isWiderType) (e1, newE2) else (e1, e2) } - // Decimal * cast int. - if (!isPromoteCast(left)) { + + if (!BackendsApiManager.getSettings.rescaleDecimalIntegralExpression()) { + (left, right) + } else if (!isPromoteCast(left) && isPromoteCastIntegral(right)) { // Have removed PromotePrecision(Cast(DecimalType)). - if (isPromoteCastIntegral(right)) { - val newRight = rescaleCastForOneSide(right) - val isWiderType = checkIsWiderType( - left.dataType.asInstanceOf[DecimalType], - newRight.dataType.asInstanceOf[DecimalType], - right.dataType.asInstanceOf[DecimalType]) - if (isWiderType) { - (left, newRight) - } else { - (left, right) - } - } else { - (left, right) - } + // Decimal * cast int. + doScale(left, right) + } else if (!isPromoteCast(right) && isPromoteCastIntegral(left)) { // Cast int * decimal. - } else if (!isPromoteCast(right)) { - if (isPromoteCastIntegral(left)) { - val newLeft = rescaleCastForOneSide(left) - val isWiderType = checkIsWiderType( - newLeft.dataType.asInstanceOf[DecimalType], - right.dataType.asInstanceOf[DecimalType], - left.dataType.asInstanceOf[DecimalType]) - if (isWiderType) { - (newLeft, right) - } else { - (left, right) - } - } else { - (left, right) - } + val (r, l) = doScale(right, left) + (l, r) } else { - // Cast int * cast int. Usually user defined cast. (left, right) } } @@ -235,6 +216,7 @@ object DecimalArithmeticUtil { } } + @tailrec def getResultType(transformer: ExpressionTransformer): Option[DecimalType] = { transformer match { case ChildTransformer(child) => @@ -289,4 +271,15 @@ object DecimalArithmeticUtil { val widerType = DecimalPrecision.widerDecimalType(left, right) widerType.equals(wider) } + + def checkAllowDecimalArithmetic(): Unit = { + // PrecisionLoss=true: velox support / ch not support + // PrecisionLoss=false: velox not support / ch support + // TODO ch support PrecisionLoss=true + if (!BackendsApiManager.getSettings.allowDecimalArithmetic) { + throw new GlutenNotSupportException( + s"Not support ${SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key} " + + s"${conf.decimalOperationsAllowPrecisionLoss} mode") + } + } } From 18a915181c6816abb7295d03788fd2a4aadf56a4 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 8 May 2024 16:25:02 +0800 Subject: [PATCH 024/402] [VL] Fix clang-format version (#5650) --- CONTRIBUTING.md | 2 +- dev/formatcppcode.sh | 4 ++-- docs/developers/CppCodingStyle.md | 2 +- docs/developers/NewToGluten.md | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9450191dd4cb..67a0f235ad8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ Developer can import the code style setting to IDE and format Java/Scala code wi ##### C/C++ code style There are some code style conventions need to comply. See [CppCodingStyle.md](https://github.com/apache/incubator-gluten/blob/main/docs/developers/CppCodingStyle.md). -For Velox backend, developer can just execute `dev/formatcppcode.sh` to format C/C++ code. It requires `clang-format-12` +For Velox backend, developer can just execute `dev/formatcppcode.sh` to format C/C++ code. It requires `clang-format-15` installed in your development env. ### License Header diff --git a/dev/formatcppcode.sh b/dev/formatcppcode.sh index 93a6107bf0ac..d391235dac9d 100755 --- a/dev/formatcppcode.sh +++ b/dev/formatcppcode.sh @@ -1,3 +1,3 @@ cd `dirname $0` -find ../cpp/core -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-12 -style=file -i {} \; -find ../cpp/velox -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-12 -style=file -i {} \; +find ../cpp/core -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-15 -style=file -i {} \; +find ../cpp/velox -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-15 -style=file -i {} \; diff --git a/docs/developers/CppCodingStyle.md b/docs/developers/CppCodingStyle.md index 5b37c2cd5012..9dca4cf69fbc 100644 --- a/docs/developers/CppCodingStyle.md +++ b/docs/developers/CppCodingStyle.md @@ -30,7 +30,7 @@ Gluten CPP coding, there are a few Philosophical rules as the following. Many aspects of C++ coding style will be covered by clang-format, such as spacing, line width, indentation and ordering (for includes, using directives and etc).  -* Always ensure your code is compatible with clang-format-12 for Velox backend. +* Always ensure your code is compatible with clang-format-15 for Velox backend. * `dev/formatcppcode.sh` is provided for formatting Velox CPP code. ## Naming Conventions diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index b3f05a64b3e8..de0ea714bd9b 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -283,16 +283,16 @@ Search `update` in Manage->Settings to turn off update mode ### Clang format -Now gluten uses clang-format 12 to format source files. +Now gluten uses clang-format 15 to format source files. ```bash -apt-get install clang-format-12 +apt-get install clang-format-15 ``` Set config in `settings.json` ```json -"clang-format.executable": "clang-format-12", +"clang-format.executable": "clang-format-15", "editor.formatOnSave": true, ``` From 537a702a0cdd8c2e54646910d7a157f0f20db42c Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Wed, 8 May 2024 13:58:00 +0530 Subject: [PATCH 025/402] [VL] Add test for getbit Spark function (#5633) --- .../gluten/execution/ScalarFunctionsValidateSuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 200c5f55147d..233a1ca96b6f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -35,10 +35,13 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } - test("Test bit_get function") { + test("Test bit_get and getbit function") { runQueryAndCompare("SELECT bit_get(l_partkey, 0) from lineitem limit 1") { checkGlutenOperatorMatch[ProjectExecTransformer] } + runQueryAndCompare("SELECT getbit(l_partkey, 0) from lineitem limit 1") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } } test("Test chr function") { From d9fe381dd8b9e1ed3658c68b382ef96f91a645f3 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Wed, 8 May 2024 16:51:56 +0800 Subject: [PATCH 026/402] [GLUTEN-5613][CH] Fix CH function SparkCheckoverflow return type not equals with spark (#5614) * Fix CH function SparkCheckoverflow return type not equals with spark * fix style --- .../SparkFunctionCheckDecimalOverflow.cpp | 277 +++++++++++------- 1 file changed, 172 insertions(+), 105 deletions(-) diff --git a/cpp-ch/local-engine/Functions/SparkFunctionCheckDecimalOverflow.cpp b/cpp-ch/local-engine/Functions/SparkFunctionCheckDecimalOverflow.cpp index 2f4652a1fee7..c75d25b6ef80 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionCheckDecimalOverflow.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionCheckDecimalOverflow.cpp @@ -15,24 +15,24 @@ * limitations under the License. */ #include "SparkFunctionCheckDecimalOverflow.h" -#include + #include #include #include -#include +#include +#include #include #include #include - namespace DB { namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; - extern const int TYPE_MISMATCH; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +extern const int TYPE_MISMATCH; } } @@ -58,137 +58,204 @@ enum class CheckExceptionMode namespace { - /// Returns received decimal value if and Decimal value has less digits then it's Precision allow, 0 otherwise. - /// Precision could be set as second argument or omitted. If omitted function uses Decimal precision of the first argument. - template - class FunctionCheckDecimalOverflow : public IFunction - { - public: - static constexpr auto name = Name::name; - static constexpr auto exception_mode = mode; +/// Returns received decimal value if and Decimal value has less digits then it's Precision allow, 0 otherwise. +/// Precision could be set as second argument or omitted. If omitted function uses Decimal precision of the first argument. +template +class FunctionCheckDecimalOverflow : public IFunction +{ +public: + static constexpr auto name = Name::name; + static constexpr auto exception_mode = mode; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 3; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 3; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!isDecimal(arguments[0]) || !isInteger(arguments[1]) || !isInteger(arguments[2])) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} {} {} of argument of function {}", - arguments[0]->getName(), - arguments[1]->getName(), - arguments[2]->getName(), - getName()); + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!isDecimal(arguments[0].type) || !isInteger(arguments[1].type) || !isInteger(arguments[2].type)) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} {} {} of argument of function {}", + arguments[0].type->getName(), + arguments[1].type->getName(), + arguments[2].type->getName(), + getName()); - if constexpr (exception_mode == CheckExceptionMode::Null) - { - if (!arguments[0]->isNullable()) - return std::make_shared(arguments[0]); - } + UInt32 precision = extractArgument(arguments[1]); + UInt32 scale = extractArgument(arguments[2]); - return arguments[0]; + auto return_type = createDecimal(precision, scale); + if constexpr (exception_mode == CheckExceptionMode::Null) + { + if (!arguments[0].type->isNullable()) + return std::make_shared(return_type); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - const auto & src_column = arguments[0]; - UInt32 precision = extractArgument(arguments[1]); - UInt32 scale = extractArgument(arguments[2]); + return return_type; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src_column = arguments[0]; + UInt32 precision = extractArgument(arguments[1]); + UInt32 scale = extractArgument(arguments[2]); + + ColumnPtr result_column; - ColumnPtr result_column; + auto call = [&](const auto & types) -> bool + { + using Types = std::decay_t; + using FromDataType = typename Types::LeftType; + using ToDataType = typename Types::RightType; - auto call = [&](const auto & types) -> bool + if constexpr (IsDataTypeDecimal) { - using Types = std::decay_t; - using Type = typename Types::RightType; - using ColVecType = ColumnDecimal; + using FromFieldType = typename FromDataType::FieldType; + using ColVecType = ColumnDecimal; if (const ColVecType * col_vec = checkAndGetColumn(src_column.column.get())) { - executeInternal(*col_vec, result_column, input_rows_count, precision, scale); + executeInternal(*col_vec, result_column, input_rows_count, precision, scale); return true; } + } + + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column while execute function {}", getName()); + }; + + if (precision <= DecimalUtils::max_precision) + callOnIndexAndDataType>(src_column.type->getTypeId(), call); + else if (precision <= DecimalUtils::max_precision) + callOnIndexAndDataType>(src_column.type->getTypeId(), call); + else if (precision <= DecimalUtils::max_precision) + callOnIndexAndDataType>(src_column.type->getTypeId(), call); + else + callOnIndexAndDataType>(src_column.type->getTypeId(), call); - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column while execute function {}", getName()); - }; - callOnBasicType(src_column.type->getTypeId(), call); - if (!result_column) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Wrong call for {} with {}", getName(), src_column.type->getName()); + if (!result_column) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Wrong call for {} with {}", getName(), src_column.type->getName()); - return result_column; + return result_column; + } + +private: + template + static void executeInternal( + const ColumnDecimal & col_source, ColumnPtr & result_column, size_t input_rows_count, UInt32 precision, UInt32 scale_to) + { + using ToFieldType = typename ToDataType::FieldType; + using ToColumnType = typename ToDataType::ColumnType; + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to [[maybe_unused]] = nullptr; + auto scale_from = col_source.getScale(); + + if constexpr (exception_mode == CheckExceptionMode::Null) + { + col_null_map_to = ColumnUInt8::create(input_rows_count, false); + vec_null_map_to = &col_null_map_to->getData(); } - private: - template - static void executeInternal( - const ColumnDecimal & col_source, ColumnPtr & result_column, size_t input_rows_count, UInt32 precision, UInt32 scale_to) + typename ToColumnType::MutablePtr col_to = ToColumnType::create(input_rows_count, scale_to); + auto & vec_to = col_to->getData(); + vec_to.resize_exact(input_rows_count); + + auto & datas = col_source.getData(); + for (size_t i = 0; i < input_rows_count; ++i) { - ColumnUInt8::MutablePtr col_null_map_to; - ColumnUInt8::Container * vec_null_map_to [[maybe_unused]] = nullptr; - auto scale_from = col_source.getScale(); + // bool overflow = outOfDigits(datas[i], precision, scale_from, scale_to); + ToFieldType result; + bool success = convertToDecimalImpl(datas[i], precision, scale_from, scale_to, result); - if constexpr (exception_mode == CheckExceptionMode::Null) + if (success) + vec_to[i] = static_cast(result); + else { - col_null_map_to = ColumnUInt8::create(input_rows_count, false); - vec_null_map_to = &col_null_map_to->getData(); + vec_to[i] = static_cast(0); + if constexpr (exception_mode == CheckExceptionMode::Null) + (*vec_null_map_to)[i] = static_cast(1); + else + throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal value is overflow."); } + } - auto & datas = col_source.getData(); - for (size_t i = 0; i < input_rows_count; ++i) - { - bool overflow = outOfDigits(datas[i], precision, scale_from, scale_to); - if (overflow) - { - if constexpr (exception_mode == CheckExceptionMode::Null) - (*vec_null_map_to)[i] = overflow; - else - throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal value is overflow."); - } - } + if constexpr (exception_mode == CheckExceptionMode::Null) + result_column = ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); + else + result_column = std::move(col_to); + } + + template + requires(IsDataTypeDecimal) + static bool convertToDecimalImpl( + const FromFieldType & decimal, UInt32 precision_to, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result) + { + if constexpr (std::is_same_v) + return convertDecimalsImpl, ToDataType>(decimal, precision_to, scale_from, scale_to, result); + + else if constexpr (std::is_same_v) + return convertDecimalsImpl, ToDataType>(decimal, precision_to, scale_from, scale_to, result); + else if constexpr (std::is_same_v) + return convertDecimalsImpl, ToDataType>(decimal, precision_to, scale_from, scale_to, result); + else + return convertDecimalsImpl, ToDataType>(decimal, precision_to, scale_from, scale_to, result); + } + + template + requires(IsDataTypeDecimal && IsDataTypeDecimal) + static bool convertDecimalsImpl( + const typename FromDataType::FieldType & value, + UInt32 precision_to, + UInt32 scale_from, + UInt32 scale_to, + typename ToDataType::FieldType & result) + { + using FromFieldType = typename FromDataType::FieldType; + using ToFieldType = typename ToDataType::FieldType; + using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>; + using MaxNativeType = typename MaxFieldType::NativeType; - typename ColumnDecimal::MutablePtr col_to = ColumnDecimal::create(std::move(col_source)); + + auto false_value = []() -> bool + { if constexpr (exception_mode == CheckExceptionMode::Null) - result_column = ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); + return false; else - result_column = std::move(col_to); - } + throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal value is overflow."); + }; - template - static bool outOfDigits(T decimal, UInt32 precision_to, UInt32 scale_from, UInt32 scale_to) + MaxNativeType converted_value; + if (scale_to > scale_from) { - using NativeT = typename T::NativeType; + converted_value = DecimalUtils::scaleMultiplier(scale_to - scale_from); + if (common::mulOverflow(static_cast(value.value), converted_value, converted_value)) + return false_value(); + } + else if (scale_to == scale_from) + converted_value = value.value; + else + converted_value = value.value / DecimalUtils::scaleMultiplier(scale_from - scale_to); - NativeT converted_value; - if (scale_to > scale_from) - { - converted_value = DecimalUtils::scaleMultiplier(scale_to - scale_from); - if (common::mulOverflow(static_cast(decimal.value), converted_value, converted_value)) - { - if constexpr (exception_mode == CheckExceptionMode::Null) - return false; - else - throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal value is overflow."); - } - } - else - converted_value = decimal.value / DecimalUtils::scaleMultiplier(scale_from - scale_to); + // if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) + // { + MaxNativeType pow10 = intExp10OfSize(precision_to); + if (converted_value <= -pow10 || converted_value >= pow10) + return false_value(); + // } - NativeT pow10 = intExp10OfSize(precision_to); - if (converted_value < 0) - return converted_value <= -pow10; - return converted_value >= pow10; - } - }; + result = static_cast(converted_value); + return true; + } +}; - using FunctionCheckDecimalOverflowThrow = FunctionCheckDecimalOverflow; - using FunctionCheckDecimalOverflowOrNull = FunctionCheckDecimalOverflow; +using FunctionCheckDecimalOverflowThrow = FunctionCheckDecimalOverflow; +using FunctionCheckDecimalOverflowOrNull = FunctionCheckDecimalOverflow; } REGISTER_FUNCTION(CheckDecimalOverflowSpark) From 32775f80b5a3f88f3a8641bf89e4efb2c55b6375 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Wed, 8 May 2024 19:40:41 +0800 Subject: [PATCH 027/402] [GLUTEN-5414][VL] FEAT: Support read CSV (#5447) The PR use Arrow's CSV reader to parse the CSV file then feed the arrow format data into Velox pipeline. This feature should be enabled with `spark.gluten.sql.native.arrow.reader.enabled = true ` As of this patch there are limitation on CSV reading, the `docs/velox-backend-limitations.md` is updated to include these limitations. --- .github/workflows/velox_docker.yml | 120 ++++--- .github/workflows/velox_velox_ut.yml | 8 +- .../clickhouse/CHSparkPlanExecApi.scala | 4 + .../backendsapi/velox/VeloxBackend.scala | 4 + .../velox/VeloxSparkPlanExecApi.scala | 5 + .../datasource/ArrowCSVFileFormat.scala | 338 ++++++++++++++++++ .../datasource/ArrowConvertorRule.scala | 85 +++++ .../apache/gluten/fs/ArrowFilesystemTest.java | 106 ++++++ .../org/apache/gluten/fs/CsvWriteSupport.java | 52 +++ .../org/apache/gluten/fs/TestDataset.java | 117 ++++++ .../apache/gluten/fs/TestNativeDataset.java | 31 ++ .../test/resources/datasource/csv/student.csv | 4 + .../gluten/execution/TestOperator.scala | 46 ++- docs/Configuration.md | 1 + docs/developers/NewToGluten.md | 42 +++ docs/velox-backend-limitations.md | 3 + .../src/build_arrow_deps_centos8.sh | 45 +++ ep/build-velox/src/build_velox.sh | 13 +- ep/build-velox/src/modify_arrow.patch | 16 +- .../backendsapi/BackendSettingsApi.scala | 2 + .../gluten/backendsapi/SparkPlanExecApi.scala | 2 + .../extension/OthersExtensionOverrides.scala | 3 + .../columnar/OffloadSingleNode.scala | 10 +- .../org/apache/gluten/utils/PlanUtil.scala | 1 + .../execution/ArrowFileSourceScanExec.scala | 65 ++++ gluten-data/pom.xml | 54 +++ .../exception/SchemaMismatchException.java | 23 ++ .../vectorized/ArrowColumnVectorUtils.java | 127 +++++++ .../org/apache/gluten/utils/ArrowUtil.scala | 222 ++++++++++++ .../catalyst/csv/CSVHeaderCheckerHelper.scala | 28 ++ .../spark/sql/utils/SparkSchemaUtil.scala | 4 + .../utils/velox/VeloxTestSettings.scala | 20 ++ .../utils/velox/VeloxTestSettings.scala | 21 ++ .../utils/velox/VeloxTestSettings.scala | 21 ++ .../utils/velox/VeloxTestSettings.scala | 20 ++ .../datasources/GlutenReadSchemaSuite.scala | 19 +- .../datasources/csv/GlutenCSVSuite.scala | 70 +++- package/pom.xml | 3 +- .../org/apache/gluten/GlutenConfig.scala | 9 + .../apache/gluten/sql/shims/SparkShims.scala | 3 + .../sql/shims/spark32/Spark32Shims.scala | 10 + .../execution/FileSourceScanExecShim.scala | 18 +- .../sql/shims/spark33/Spark33Shims.scala | 11 + .../execution/FileSourceScanExecShim.scala | 18 +- .../sql/shims/spark34/Spark34Shims.scala | 12 +- .../execution/FileSourceScanExecShim.scala | 26 +- .../sql/shims/spark35/Spark35Shims.scala | 12 +- .../execution/FileSourceScanExecShim.scala | 26 +- 48 files changed, 1828 insertions(+), 72 deletions(-) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala create mode 100644 backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java create mode 100644 backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java create mode 100644 backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java create mode 100644 backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java create mode 100644 backends-velox/src/test/resources/datasource/csv/student.csv create mode 100755 ep/build-velox/src/build_arrow_deps_centos8.sh create mode 100644 gluten-core/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala create mode 100644 gluten-data/src/main/java/org/apache/gluten/exception/SchemaMismatchException.java create mode 100644 gluten-data/src/main/java/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorUtils.java create mode 100644 gluten-data/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderCheckerHelper.scala diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 19ff0cd48685..284bf9198862 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -500,12 +500,20 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Build Gluten velox third party + - name: Setup java and maven run: | yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven + - name: Build Gluten velox third party + run: | + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | @@ -516,11 +524,6 @@ jobs: run: | cd $GITHUB_WORKSPACE/cpp/build && \ ctest -V - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.2.2 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -563,23 +566,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 + - name: Setup java and maven + run: | + yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven - name: Build Gluten velox third party run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.2.2 (slow tests) run: | cd $GITHUB_WORKSPACE// && \ @@ -599,23 +605,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Build Gluten velox third party + - name: Setup java and maven run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven + - name: Build Gluten velox third party + run: | + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.3.1 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -654,23 +663,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 + - name: Setup java and maven + run: | + yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven - name: Build Gluten velox third party run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.3.1 (slow tests) run: | cd $GITHUB_WORKSPACE// && \ @@ -690,23 +702,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Build Gluten velox third party + - name: Setup java and maven run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven + - name: Build Gluten velox third party + run: | + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.4.2 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -745,23 +760,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Build Gluten velox third party + - name: Setup java and maven run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven + - name: Build Gluten velox third party + run: | + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.4.2 (slow tests) run: | cd $GITHUB_WORKSPACE// && \ @@ -781,23 +799,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Build Gluten velox third party + - name: Setup java and maven run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven + - name: Build Gluten velox third party + run: | + export MAVEN_HOME=/usr/lib/maven && \ + export PATH=${PATH}:${MAVEN_HOME}/bin && \ cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.5.1 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -835,23 +856,26 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Build Gluten velox third party + - name: Setup java and maven run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven + - name: Build Gluten velox third party + run: | + export MAVEN_HOME=/usr/lib/maven + export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ + ./build_arrow_deps_centos8.sh && \ ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - name: Build Gluten CPP library run: | cd $GITHUB_WORKSPACE/cpp && \ source /opt/rh/gcc-toolset-9/enable && \ ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - - name: Setup java and maven - run: | - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - name: Prepare spark.test.home for Spark 3.5.1 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ diff --git a/.github/workflows/velox_velox_ut.yml b/.github/workflows/velox_velox_ut.yml index 53a90cdc1720..f1a1f9371d6e 100644 --- a/.github/workflows/velox_velox_ut.yml +++ b/.github/workflows/velox_velox_ut.yml @@ -19,9 +19,10 @@ name: Velox backend Velox Unit test on: pull_request: paths: - - '.github/workflows/velox_velox_ut.yml' + # TODO: wait to fix + # - '.github/workflows/velox_velox_ut.yml' - 'dev/**' - - 'ep/**' #get_velox change + # - 'ep/**' #get_velox change concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -64,4 +65,5 @@ jobs: - name: Run Tests run: | - cd ${GITHUB_WORKSPACE}/ep/build-velox/build/velox_ep/_build/release && ctest -E velox_cache_test -j 4 --output-on-failure --no-tests=error \ No newline at end of file + ccache -c + cd ${GITHUB_WORKSPACE}/ep/build-velox/build/velox_ep/_build/release && ctest -E velox_cache_test -j 4 --output-on-failure --no-tests=error diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index a9a12a3ea2ce..030648b06fa6 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -617,6 +617,10 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { override def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] = List(spark => NativeWritePostRule(spark)) + override def genInjectPostHocResolutionRules(): List[SparkSession => Rule[LogicalPlan]] = { + List() + } + /** * Generate extended Strategies. * diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 00b67fca7500..0d38fd07c570 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -487,6 +487,10 @@ object VeloxBackendSettings extends BackendSettingsApi { ) } + override def enableNativeArrowReadFiles(): Boolean = { + GlutenConfig.getConf.enableNativeArrowReader + } + override def shouldRewriteCount(): Boolean = { // Velox backend does not support count if it has more that one child, // so we should rewrite it. diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 0a9f3ef65fd1..f98055630304 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -18,6 +18,7 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.SparkPlanExecApi +import org.apache.gluten.datasource.ArrowConvertorRule import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ @@ -767,6 +768,10 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { SparkShimLoader.getSparkShims.getExtendedColumnarPostRules() ::: List() } + override def genInjectPostHocResolutionRules(): List[SparkSession => Rule[LogicalPlan]] = { + List(ArrowConvertorRule) + } + /** * Generate extended Strategy. * diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala new file mode 100644 index 000000000000..c05af24ff611 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource + +import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.exception.SchemaMismatchException +import org.apache.gluten.execution.RowToVeloxColumnarExec +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.utils.{ArrowUtil, Iterators} +import org.apache.gluten.vectorized.ArrowWritableColumnVector + +import org.apache.spark.TaskContext +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.csv.{CSVHeaderChecker, CSVHeaderCheckerHelper, CSVOptions, UnivocityParser} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, JoinedRow} +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection +import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFileLinesReader, OutputWriterFactory, PartitionedFile} +import org.apache.spark.sql.execution.datasources.csv.CSVDataSource +import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.{DataSourceRegister, Filter} +import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.SerializableConfiguration + +import org.apache.arrow.dataset.file.FileSystemDatasetFactory +import org.apache.arrow.dataset.scanner.ScanOptions +import org.apache.arrow.vector.VectorUnloader +import org.apache.arrow.vector.types.pojo.Schema +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} + +import java.net.URLDecoder +import java.util.Optional + +import scala.collection.JavaConverters.asScalaBufferConverter + +class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging with Serializable { + + private val fileFormat = org.apache.arrow.dataset.file.FileFormat.CSV + + override def isSplitable( + sparkSession: SparkSession, + options: Map[String, String], + path: Path): Boolean = { + false + } + + override def inferSchema( + sparkSession: SparkSession, + options: Map[String, String], + files: Seq[FileStatus]): Option[StructType] = { + ArrowUtil.readSchema(files, fileFormat) + } + + override def supportBatch(sparkSession: SparkSession, dataSchema: StructType): Boolean = true + + private def checkHeader( + file: PartitionedFile, + dataSchema: StructType, + requiredSchema: StructType, + parsedOptions: CSVOptions, + actualFilters: Seq[Filter], + conf: Configuration): Unit = { + val isStartOfFile = file.start == 0 + if (!isStartOfFile) { + return + } + val actualDataSchema = StructType( + dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val actualRequiredSchema = StructType( + requiredSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val parser = + new UnivocityParser(actualDataSchema, actualRequiredSchema, parsedOptions, actualFilters) + val schema = if (parsedOptions.columnPruning) actualRequiredSchema else actualDataSchema + val headerChecker = new CSVHeaderChecker( + schema, + parsedOptions, + source = s"CSV file: ${file.filePath}", + isStartOfFile) + + val lines = { + val linesReader = + new HadoopFileLinesReader(file, parser.options.lineSeparatorInRead, conf) + Option(TaskContext.get()) + .foreach(_.addTaskCompletionListener[Unit](_ => linesReader.close())) + linesReader.map { + line => new String(line.getBytes, 0, line.getLength, parser.options.charset) + } + } + CSVHeaderCheckerHelper.checkHeaderColumnNames(headerChecker, lines, parser.tokenizer) + } + + private def readArrow( + file: PartitionedFile, + actualReadFields: Schema, + caseSensitive: Boolean, + requiredSchema: StructType, + partitionSchema: StructType, + factory: FileSystemDatasetFactory, + batchSize: Int): Iterator[InternalRow] = { + val compare = ArrowUtil.compareStringFunc(caseSensitive) + val actualReadFieldNames = actualReadFields.getFields.asScala.map(_.getName).toArray + val actualReadSchema = new StructType( + actualReadFieldNames.map(f => requiredSchema.find(field => compare(f, field.name)).get)) + val dataset = factory.finish(actualReadFields) + + val hasMissingColumns = actualReadFields.getFields.size() != requiredSchema.size + + val scanOptions = new ScanOptions(batchSize, Optional.of(actualReadFieldNames)) + val scanner = dataset.newScan(scanOptions) + + val partitionVectors = + ArrowUtil.loadPartitionColumns(batchSize, partitionSchema, file.partitionValues) + + val nullVectors = if (hasMissingColumns) { + val missingSchema = + new StructType(requiredSchema.filterNot(actualReadSchema.contains).toArray) + ArrowUtil.loadMissingColumns(batchSize, missingSchema) + } else { + Array.empty[ArrowWritableColumnVector] + } + val reader = scanner.scanBatches() + Iterators + .wrap(new Iterator[ColumnarBatch] { + + override def hasNext: Boolean = { + reader.loadNextBatch() + } + + override def next: ColumnarBatch = { + val root = reader.getVectorSchemaRoot + val unloader = new VectorUnloader(root) + val batch = ArrowUtil.loadBatch( + unloader.getRecordBatch, + actualReadSchema, + requiredSchema, + partitionVectors, + nullVectors) + batch + } + }) + .recycleIterator { + scanner.close() + dataset.close() + factory.close() + reader.close() + partitionVectors.foreach(_.close()) + nullVectors.foreach(_.close()) + } + .recyclePayload(_.close()) + .create() + .asInstanceOf[Iterator[InternalRow]] + } + + private def rowToColumn( + schema: StructType, + batchSize: Int, + it: Iterator[InternalRow]): Iterator[InternalRow] = { + // note, these metrics are unused but just make `RowToVeloxColumnarExec` happy + val numInputRows = new SQLMetric("numInputRows") + val numOutputBatches = new SQLMetric("numOutputBatches") + val convertTime = new SQLMetric("convertTime") + val veloxBatch = RowToVeloxColumnarExec.toColumnarBatchIterator( + it, + schema, + numInputRows, + numOutputBatches, + convertTime, + batchSize + ) + veloxBatch + .map(v => ColumnarBatches.ensureLoaded(ArrowBufferAllocators.contextInstance(), v)) + .asInstanceOf[Iterator[InternalRow]] + } + + private def toAttribute(field: StructField): AttributeReference = + AttributeReference(field.name, field.dataType, field.nullable, field.metadata)() + + private def toAttributes(schema: StructType): Seq[AttributeReference] = { + schema.map(toAttribute) + } + + private def withPartitionValue( + requiredSchema: StructType, + partitionSchema: StructType, + iter: Iterator[InternalRow], + file: PartitionedFile): (StructType, Iterator[InternalRow]) = { + val fullSchema = toAttributes(requiredSchema) ++ toAttributes(partitionSchema) + + // Using lazy val to avoid serialization + lazy val appendPartitionColumns = + GenerateUnsafeProjection.generate(fullSchema, fullSchema) + // Using local val to avoid per-row lazy val check (pre-mature optimization?...) + val converter = appendPartitionColumns + + // Note that we have to apply the converter even though `file.partitionValues` is empty. + // This is because the converter is also responsible for converting safe `InternalRow`s into + // `UnsafeRow`s. + if (partitionSchema.isEmpty) { + val rows = iter.map(dataRow => converter(dataRow)) + (StructType(requiredSchema ++ partitionSchema), rows) + } else { + val joinedRow = new JoinedRow() + val rows = iter.map(dataRow => converter(joinedRow(dataRow, file.partitionValues))) + (StructType(requiredSchema ++ partitionSchema), rows) + } + } + + private def fallbackReadVanilla( + dataSchema: StructType, + requiredSchema: StructType, + conf: Configuration, + parsedOptions: CSVOptions, + file: PartitionedFile, + actualFilters: Seq[Filter], + columnPruning: Boolean): Iterator[InternalRow] = { + val actualDataSchema = StructType( + dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val actualRequiredSchema = StructType( + requiredSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val parser = + new UnivocityParser(actualDataSchema, actualRequiredSchema, parsedOptions, actualFilters) + val schema = if (columnPruning) actualRequiredSchema else actualDataSchema + val isStartOfFile = file.start == 0 + val headerChecker = new CSVHeaderChecker( + schema, + parsedOptions, + source = s"CSV file: ${file.filePath}", + isStartOfFile) + CSVDataSource(parsedOptions).readFile(conf, file, parser, headerChecker, requiredSchema) + } + + override def buildReaderWithPartitionValues( + sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { + val sqlConf = sparkSession.sessionState.conf + val broadcastedHadoopConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) + val batchSize = sqlConf.columnBatchSize + val caseSensitive = sqlConf.caseSensitiveAnalysis + val columnPruning = sqlConf.csvColumnPruning && + !requiredSchema.exists(_.name == sparkSession.sessionState.conf.columnNameOfCorruptRecord) + val parsedOptions = new CSVOptions( + options, + columnPruning, + sparkSession.sessionState.conf.sessionLocalTimeZone, + sparkSession.sessionState.conf.columnNameOfCorruptRecord) + val actualFilters = + filters.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord)) + (file: PartitionedFile) => { + checkHeader( + file, + dataSchema, + requiredSchema, + parsedOptions, + actualFilters, + broadcastedHadoopConf.value.value) + val factory = + ArrowUtil.makeArrowDiscovery(URLDecoder.decode(file.filePath.toString, "UTF-8"), fileFormat) + // todo predicate validation / pushdown + val fileFields = factory.inspect().getFields.asScala + // TODO: support array/map/struct types in out-of-order schema reading. + try { + val actualReadFields = + ArrowUtil.getRequestedField(requiredSchema, fileFields, caseSensitive) + readArrow( + file, + actualReadFields, + caseSensitive, + requiredSchema, + partitionSchema, + factory, + batchSize) + } catch { + case e: SchemaMismatchException => + logWarning(e.getMessage) + val iter = fallbackReadVanilla( + dataSchema, + requiredSchema, + broadcastedHadoopConf.value.value, + parsedOptions, + file, + actualFilters, + columnPruning) + val (schema, rows) = withPartitionValue(requiredSchema, partitionSchema, iter, file) + rowToColumn(schema, batchSize, rows) + case d: Exception => throw d + } + + } + } + + override def vectorTypes( + requiredSchema: StructType, + partitionSchema: StructType, + sqlConf: SQLConf): Option[Seq[String]] = { + Option( + Seq.fill(requiredSchema.fields.length + partitionSchema.fields.length)( + classOf[ArrowWritableColumnVector].getName + )) + } + + override def shortName(): String = "arrowcsv" + + override def hashCode(): Int = getClass.hashCode() + + override def equals(other: Any): Boolean = other.isInstanceOf[ArrowCSVFileFormat] + + override def prepareWrite( + sparkSession: SparkSession, + job: _root_.org.apache.hadoop.mapreduce.Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + throw new UnsupportedOperationException() + } +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala new file mode 100644 index 000000000000..e29313a3809e --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource + +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.sql.shims.SparkShimLoader + +import org.apache.spark.annotation.Experimental +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.csv.CSVOptions +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.util.PermissiveMode +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} +import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.utils.SparkSchemaUtil + +import java.nio.charset.StandardCharsets + +@Experimental +case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { + override def apply(plan: LogicalPlan): LogicalPlan = { + if (!BackendsApiManager.getSettings.enableNativeArrowReadFiles()) { + return plan + } + plan.resolveOperators { + // Read path + case l @ LogicalRelation( + r @ HadoopFsRelation(_, _, dataSchema, _, _: CSVFileFormat, options), + _, + _, + _) => + val csvOptions = new CSVOptions( + options, + columnPruning = session.sessionState.conf.csvColumnPruning, + session.sessionState.conf.sessionLocalTimeZone) + if ( + checkSchema(dataSchema) && + checkCsvOptions(csvOptions, session.sessionState.conf.sessionLocalTimeZone) + ) { + l.copy(relation = r.copy(fileFormat = new ArrowCSVFileFormat())(session)) + } else l + case r => + r + } + } + + private def checkCsvOptions(csvOptions: CSVOptions, timeZone: String): Boolean = { + csvOptions.headerFlag && !csvOptions.multiLine && csvOptions.delimiter == "," && + csvOptions.quote == '\"' && + csvOptions.escape == '\\' && + csvOptions.lineSeparator.isEmpty && + csvOptions.charset == StandardCharsets.UTF_8.name() && + csvOptions.parseMode == PermissiveMode && !csvOptions.inferSchemaFlag && + csvOptions.nullValue == "" && + csvOptions.emptyValueInRead == "" && csvOptions.comment == '\u0000' && + csvOptions.columnPruning && + SparkShimLoader.getSparkShims.dateTimestampFormatInReadIsDefaultValue(csvOptions, timeZone) + } + + private def checkSchema(schema: StructType): Boolean = { + try { + SparkSchemaUtil.toArrowSchema(schema) + true + } catch { + case _: Exception => + false + } + } +} diff --git a/backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java b/backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java new file mode 100644 index 000000000000..5dd29856ac62 --- /dev/null +++ b/backends-velox/src/test/java/org/apache/gluten/fs/ArrowFilesystemTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.fs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.primitives.Primitives; +import org.apache.arrow.dataset.file.FileFormat; +import org.apache.arrow.dataset.file.FileSystemDatasetFactory; +import org.apache.arrow.dataset.jni.NativeMemoryPool; +import org.apache.arrow.dataset.scanner.ScanOptions; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class ArrowFilesystemTest extends TestNativeDataset { + + @ClassRule public static final TemporaryFolder TMP = new TemporaryFolder(); + + private void checkParquetReadResult( + Schema schema, String expectedJson, List actual) throws IOException { + final ObjectMapper json = new ObjectMapper(); + final Set expectedSet = json.readValue(expectedJson, Set.class); + final Set> actualSet = new HashSet<>(); + final int fieldCount = schema.getFields().size(); + try (VectorSchemaRoot vsr = VectorSchemaRoot.create(schema, rootAllocator())) { + VectorLoader loader = new VectorLoader(vsr); + for (ArrowRecordBatch batch : actual) { + System.out.println(batch.toString()); + loader.load(batch); + int batchRowCount = vsr.getRowCount(); + for (int i = 0; i < batchRowCount; i++) { + List row = new ArrayList<>(); + for (int j = 0; j < fieldCount; j++) { + Object object = vsr.getVector(j).getObject(i); + if (Primitives.isWrapperType(object.getClass())) { + row.add(object); + } else { + row.add(object.toString()); + } + } + actualSet.add(row); + } + } + } + Assert.assertEquals( + "Mismatched data read from Parquet, actual: " + json.writeValueAsString(actualSet) + ";", + expectedSet, + actualSet); + } + + @Test + public void testBaseCsvRead() throws Exception { + CsvWriteSupport writeSupport = + CsvWriteSupport.writeTempFile( + TMP.newFolder(), "Name,Language", "Juno,Java", "Peter,Python", "Celin,C++"); + String expectedJsonUnordered = + "[[\"Juno\", \"Java\"], [\"Peter\", \"Python\"], [\"Celin\", \"C++\"]]"; + ScanOptions options = new ScanOptions(100); + try (FileSystemDatasetFactory factory = + new FileSystemDatasetFactory( + rootAllocator(), + NativeMemoryPool.getDefault(), + FileFormat.CSV, + writeSupport.getOutputURI())) { + List datum = collectResultFromFactory(factory, options); + Schema schema = inferResultSchemaFromFactory(factory, options); + + assertScanBatchesProduced(factory, options); + assertEquals(1, datum.size()); + assertEquals(2, schema.getFields().size()); + assertEquals("Name", schema.getFields().get(0).getName()); + + checkParquetReadResult(schema, expectedJsonUnordered, datum); + + AutoCloseables.close(datum); + } + } +} diff --git a/backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java b/backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java new file mode 100644 index 000000000000..bcc1cceb9a6c --- /dev/null +++ b/backends-velox/src/test/java/org/apache/gluten/fs/CsvWriteSupport.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.fs; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Random; + +public class CsvWriteSupport { + private final URI uri; + private final Random random = new Random(); + + public CsvWriteSupport(File outputFolder) throws URISyntaxException { + uri = + new URI( + "file", + outputFolder.getPath() + File.separator + "generated-" + random.nextLong() + ".csv", + null); + } + + public static CsvWriteSupport writeTempFile(File outputFolder, String... values) + throws URISyntaxException, IOException { + CsvWriteSupport writer = new CsvWriteSupport(outputFolder); + try (FileWriter addValues = new FileWriter(new File(writer.uri), true)) { + for (Object value : values) { + addValues.write(value + "\n"); + } + } + return writer; + } + + public String getOutputURI() { + return uri.toString(); + } +} diff --git a/backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java b/backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java new file mode 100644 index 000000000000..f2a1095bb223 --- /dev/null +++ b/backends-velox/src/test/java/org/apache/gluten/fs/TestDataset.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.fs; + +import org.apache.arrow.dataset.scanner.ScanOptions; +import org.apache.arrow.dataset.scanner.Scanner; +import org.apache.arrow.dataset.source.Dataset; +import org.apache.arrow.dataset.source.DatasetFactory; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class TestDataset { + private RootAllocator allocator = null; + + @Before + public void setUp() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void tearDown() { + allocator.close(); + } + + protected RootAllocator rootAllocator() { + return allocator; + } + + protected List collectResultFromFactory( + DatasetFactory factory, ScanOptions options) { + final Dataset dataset = factory.finish(); + final Scanner scanner = dataset.newScan(options); + try { + final List ret = collectTaskData(scanner); + AutoCloseables.close(scanner, dataset); + return ret; + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected List collectTaskData(Scanner scan) { + try (ArrowReader reader = scan.scanBatches()) { + List batches = new ArrayList<>(); + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + final VectorUnloader unloader = new VectorUnloader(root); + batches.add(unloader.getRecordBatch()); + } + return batches; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + protected Schema inferResultSchemaFromFactory(DatasetFactory factory, ScanOptions options) { + final Dataset dataset = factory.finish(); + final Scanner scanner = dataset.newScan(options); + final Schema schema = scanner.schema(); + try { + AutoCloseables.close(scanner, dataset); + } catch (Exception e) { + throw new RuntimeException(e); + } + return schema; + } + + protected Stream stream(Iterable iterable) { + return StreamSupport.stream(iterable.spliterator(), false); + } + + protected List collect(Iterable iterable) { + return stream(iterable).collect(Collectors.toList()); + } + + protected Stream stream(Iterator iterator) { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); + } + + protected List collect(Iterator iterator) { + return stream(iterator).collect(Collectors.toList()); + } +} diff --git a/backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java b/backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java new file mode 100644 index 000000000000..b06fa3c606bd --- /dev/null +++ b/backends-velox/src/test/java/org/apache/gluten/fs/TestNativeDataset.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.fs; + +import org.apache.arrow.dataset.scanner.ScanOptions; +import org.apache.arrow.dataset.scanner.Scanner; +import org.apache.arrow.dataset.source.Dataset; +import org.apache.arrow.dataset.source.DatasetFactory; +import org.junit.Assert; + +public abstract class TestNativeDataset extends TestDataset { + protected void assertScanBatchesProduced(DatasetFactory factory, ScanOptions options) { + final Dataset dataset = factory.finish(); + final Scanner scanner = dataset.newScan(options); + Assert.assertNotNull(scanner.scanBatches()); + } +} diff --git a/backends-velox/src/test/resources/datasource/csv/student.csv b/backends-velox/src/test/resources/datasource/csv/student.csv new file mode 100644 index 000000000000..bc71daf24a60 --- /dev/null +++ b/backends-velox/src/test/resources/datasource/csv/student.csv @@ -0,0 +1,4 @@ +Name,Language +Juno,Java +Peter,Python +Celin,C++ diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 7dc8f1e0b279..a14a5b7e78de 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -17,11 +17,12 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig +import org.apache.gluten.datasource.ArrowCSVFileFormat import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, Row} -import org.apache.spark.sql.execution.{FilterExec, GenerateExec, ProjectExec, RDDScanExec} +import org.apache.spark.sql.execution.{ArrowFileSourceScanExec, ColumnarToRowExec, FilterExec, GenerateExec, ProjectExec, RDDScanExec} import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.functions.{avg, col, lit, to_date, udf} import org.apache.spark.sql.internal.SQLConf @@ -52,7 +53,8 @@ class TestOperator extends VeloxWholeStageTransformerSuite { .set("spark.memory.offHeap.size", "2g") .set("spark.unsafe.exceptionOnMemoryLeak", "true") .set("spark.sql.autoBroadcastJoinThreshold", "-1") - .set("spark.sql.sources.useV1SourceList", "avro,parquet") + .set("spark.sql.sources.useV1SourceList", "avro,parquet,csv") + .set(GlutenConfig.NATIVE_ARROW_READER_ENABLED.key, "true") } test("simple_select") { @@ -473,6 +475,46 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + test("csv scan") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + runQueryAndCompare("select * from student") { + df => + val plan = df.queryExecution.executedPlan + print(plan) + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) + val scan = plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).toList.head + assert( + scan + .asInstanceOf[ArrowFileSourceScanExec] + .relation + .fileFormat + .isInstanceOf[ArrowCSVFileFormat]) + } + } + + test("csv scan with filter") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + runQueryAndCompare("select * from student where Name = 'Peter'") { + df => + assert(df.queryExecution.executedPlan.find(s => s.isInstanceOf[ColumnarToRowExec]).isEmpty) + assert( + df.queryExecution.executedPlan + .find(s => s.isInstanceOf[ArrowFileSourceScanExec]) + .isDefined) + } + } + test("test OneRowRelation") { val df = sql("SELECT 1") checkAnswer(df, Row(1)) diff --git a/docs/Configuration.md b/docs/Configuration.md index 06a766f2d848..58ec2a12277b 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -51,6 +51,7 @@ You can add these configurations into spark-defaults.conf to enable or disable t | spark.gluten.sql.columnar.numaBinding | Set up NUMABinding, default is false | true | | spark.gluten.sql.columnar.coreRange | Set up the core range for NUMABinding, only works when numaBinding set to true.
The setting is based on the number of cores in your system. Use 72 cores as an example. | 0-17,36-53 |18-35,54-71 | | spark.gluten.sql.native.bloomFilter | Enable or Disable native runtime bloom filter. | true | +| spark.gluten.sql.native.arrow.reader.enabled | Enable or Disable native arrow read CSV file format | false | | spark.gluten.sql.columnar.wholeStage.fallback.threshold | Configure the threshold for whether whole stage will fall back in AQE supported case by counting the number of ColumnarToRow & vanilla leaf node | \>= 1 | | spark.gluten.sql.columnar.query.fallback.threshold | Configure the threshold for whether query will fall back by counting the number of ColumnarToRow & vanilla leaf node | \>= 1 | | spark.gluten.sql.columnar.fallback.ignoreRowToColumnar | When true, the fallback policy ignores the RowToColumnar when counting fallback number. | true | diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index de0ea714bd9b..1eb21d1e6c05 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -369,6 +369,48 @@ wait to attach.... (gdb) c ``` +# Debug Memory leak +## Arrow memory allocator leak +If you receive error message like + +```bash +4/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Detected leaked Arrow allocator [Default], size: 191, process accumulated leaked size: 191... +24/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Leaked allocator stack Allocator(ROOT) 0/191/319/9223372036854775807 (res/actual/peak/limit) +``` +You can open the Arrow allocator debug config by add VP option `-Darrow.memory.debug.allocator=true`, then you can get more details like +```bash +child allocators: 0 + ledgers: 7 + ledger[10] allocator: ROOT), isOwning: , size: , references: 1, life: 10483701311283711..0, allocatorManager: [, life: ] holds 1 buffers. + ArrowBuf[11], address:140100698555856, capacity:128 + event log for: ArrowBuf[11] + 10483701311362601 create() + at org.apache.arrow.memory.util.HistoricalLog$Event.(HistoricalLog.java:175) + at org.apache.arrow.memory.util.HistoricalLog.recordEvent(HistoricalLog.java:83) + at org.apache.arrow.memory.ArrowBuf.(ArrowBuf.java:97) + at org.apache.arrow.memory.BufferLedger.newArrowBuf(BufferLedger.java:271) + at org.apache.arrow.memory.BaseAllocator.bufferWithoutReservation(BaseAllocator.java:340) + at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:316) + at org.apache.arrow.memory.RootAllocator.buffer(RootAllocator.java:29) + at org.apache.arrow.memory.BaseAllocator.buffer(BaseAllocator.java:280) + at org.apache.arrow.memory.RootAllocator.buffer(RootAllocator.java:29) + at org.apache.arrow.c.ArrowArray.allocateNew(ArrowArray.java:116) + at org.apache.arrow.c.ArrayImporter.importArray(ArrayImporter.java:61) + at org.apache.arrow.c.Data.importIntoVector(Data.java:289) + at org.apache.arrow.c.Data.importIntoVectorSchemaRoot(Data.java:332) + at org.apache.arrow.dataset.jni.NativeScanner$NativeReader.loadNextBatch(NativeScanner.java:151) + at org.apache.gluten.datasource.ArrowFileFormat$$anon$1.hasNext(ArrowFileFormat.scala:99) + at org.apache.gluten.utils.IteratorCompleter.hasNext(Iterators.scala:69) + at org.apache.spark.memory.SparkMemoryUtil$UnsafeItr.hasNext(SparkMemoryUtil.scala:246) +``` +## CPP code memory leak +Sometimes you cannot get the coredump symbols, if you debug memory leak, you can write googletest to use valgrind to detect +``` +apt install valgrind +valgrind --leak-check=yes ./exec_backend_test +``` + + # Run TPC-H and TPC-DS We supply `/tools/gluten-it` to execute these queries diff --git a/docs/velox-backend-limitations.md b/docs/velox-backend-limitations.md index 73bbdf07af93..75b52f38e17a 100644 --- a/docs/velox-backend-limitations.md +++ b/docs/velox-backend-limitations.md @@ -157,3 +157,6 @@ Gluten's. - Complex types - Parquet scan of nested array with struct or array as element type is not supported in Velox (fallback behavior). - Parquet scan of nested map with struct as key type, or array type as value type is not supported in Velox (fallback behavior). + +### CSV Read +The header option should be true. And now we only support DatasourceV1, user should set this setting spark.sql.sources.useV1SourceList=csv. Not support user defined read option, will fallback to vanilla Spark in most case. Will fallback to vanilla Spark and log warning when user specifies schema is different with file schema. diff --git a/ep/build-velox/src/build_arrow_deps_centos8.sh b/ep/build-velox/src/build_arrow_deps_centos8.sh new file mode 100755 index 000000000000..8dfc2af9375e --- /dev/null +++ b/ep/build-velox/src/build_arrow_deps_centos8.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exu + +NPROC=$(getconf _NPROCESSORS_ONLN) + +function wget_and_untar { + local URL=$1 + local DIR=$2 + mkdir -p "${DIR}" + pushd "${DIR}" + curl -L "${URL}" > $2.tar.gz + tar -xz --strip-components=1 -f $2.tar.gz + popd +} + +function install_openssl { + wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl + cd openssl + ./config no-shared && make depend && make && sudo make install + cd .. +} + +function install_arrow_deps { + install_openssl +} + +# Activate gcc9; enable errors on unset variables afterwards. +# source /opt/rh/gcc-toolset-9/enable || exit 1 +install_arrow_deps +echo "All dependencies for Arrow installed!" diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index dae86e6ff6b4..3fc0dc6f661f 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -34,7 +34,7 @@ ENABLE_TESTS=OFF # Set to ON for gluten cpp test build. BUILD_TEST_UTILS=OFF RUN_SETUP_SCRIPT=ON -COMPILE_ARROW_JAVA=OFF +COMPILE_ARROW_JAVA=ON NUM_THREADS="" OTHER_ARGUMENTS="" @@ -282,6 +282,7 @@ function compile_arrow_java_module() { ARROW_INSTALL_DIR="${ARROW_HOME}/../../install" pushd $ARROW_HOME/java + mvn clean install -pl maven/module-info-compiler-maven-plugin -am \ -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip @@ -289,9 +290,15 @@ function compile_arrow_java_module() { mvn generate-resources -P generate-libs-cdata-all-os -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \ -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N + # Arrow JNI Date Interface CPP libraries + export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} + mvn generate-resources -Pgenerate-libs-jni-macos-linux -N -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR \ + -DARROW_GANDIVA=OFF -DARROW_JAVA_JNI_ENABLE_GANDIVA=OFF -DARROW_ORC=OFF -DARROW_JAVA_JNI_ENABLE_ORC=OFF \ + -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N + # Arrow Java libraries - mvn clean install -P arrow-c-data -pl c -am -DskipTests -Dcheckstyle.skip \ - -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib \ + mvn clean install -Parrow-jni -P arrow-c-data -pl dataset,c -am \ + -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.cpp.build.dir=$ARROW_INSTALL_DIR/lib \ -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip popd } diff --git a/ep/build-velox/src/modify_arrow.patch b/ep/build-velox/src/modify_arrow.patch index a6d5e832c779..e5444d28043f 100644 --- a/ep/build-velox/src/modify_arrow.patch +++ b/ep/build-velox/src/modify_arrow.patch @@ -1,8 +1,8 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -index 94f926039..f7ebf9233 100644 +index a2627c190..e453512e6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -2501,13 +2501,9 @@ if(ARROW_WITH_ZSTD) +@@ -2557,13 +2557,9 @@ if(ARROW_WITH_ZSTD) if(ZSTD_VENDORED) set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static) else() @@ -30,3 +30,15 @@ index a24f272fe..e25f78c85 100644 #include #include #include +diff --git a/java/pom.xml b/java/pom.xml +index a8328576b..53a70fab8 100644 +--- a/java/pom.xml ++++ b/java/pom.xml +@@ -1102,6 +1102,7 @@ + -DARROW_ORC=${ARROW_ORC} + -DARROW_PARQUET=${ARROW_PARQUET} + -DARROW_S3=ON ++ -DARROW_HDFS=ON + -DARROW_SUBSTRAIT=${ARROW_DATASET} + -DARROW_USE_CCACHE=ON + -DCMAKE_BUILD_TYPE=Release diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index ddf62201daeb..e02c2290ae59 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -138,6 +138,8 @@ trait BackendSettingsApi { def enableNativeWriteFiles(): Boolean + def enableNativeArrowReadFiles(): Boolean = false + def shouldRewriteCount(): Boolean = false def supportCartesianProductExec(): Boolean = false diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index f5e08a05d7a1..fb2fd961b481 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -425,6 +425,8 @@ trait SparkPlanExecApi { */ def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] + def genInjectPostHocResolutionRules(): List[SparkSession => Rule[LogicalPlan]] + def genGetStructFieldTransformer( substraitExprName: String, childTransformer: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/OthersExtensionOverrides.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/OthersExtensionOverrides.scala index d9f8fcd08051..0897f411fce5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/OthersExtensionOverrides.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/OthersExtensionOverrides.scala @@ -38,5 +38,8 @@ object OthersExtensionOverrides extends GlutenSparkExtensionsInjector { BackendsApiManager.getSparkPlanExecApiInstance .genInjectedFunctions() .foreach(extensions.injectFunction) + BackendsApiManager.getSparkPlanExecApiInstance + .genInjectPostHocResolutionRules() + .foreach(extensions.injectPostHocResolutionRule) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 84a2ec5c6ec8..067aad32cd7a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -297,7 +297,15 @@ object OffloadOthers { class ReplaceSingleNode() extends LogLevelUtil with Logging { def doReplace(p: SparkPlan): SparkPlan = { - val plan = p + val plan = p match { + case plan: FileSourceScanExec + if plan.relation.fileFormat.getClass.getSimpleName == "ArrowCSVFileFormat" => + val arrowScan = ArrowFileSourceScanExec(plan) + TransformHints.tagNotTransformable(arrowScan, "Arrow scan cannot transform") + return arrowScan + case p => p + } + if (TransformHints.isNotTransformable(plan)) { logDebug(s"Columnar Processing for ${plan.getClass} is under row guard.") plan match { diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala index 15aa11a13fd8..610f14c86024 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala @@ -50,6 +50,7 @@ object PlanUtil { case s: WholeStageCodegenExec => outputNativeColumnarData(s.child) case s: AdaptiveSparkPlanExec => outputNativeColumnarData(s.executedPlan) case i: InMemoryTableScanExec => PlanUtil.isGlutenTableCache(i) + case _: ArrowFileSourceScanExec => false case _: GlutenPlan => true case _ => false } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala new file mode 100644 index 000000000000..133bf88b3cbb --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution + +import org.apache.gluten.extension.GlutenPlan + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.vectorized.ColumnarBatch + +import scala.concurrent.duration.NANOSECONDS + +case class ArrowFileSourceScanExec(original: FileSourceScanExec) + extends ArrowFileSourceScanLikeShim(original) + with GlutenPlan { + + lazy val inputRDD: RDD[InternalRow] = original.inputRDD + + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + + override def output: Seq[Attribute] = original.output + + override def supportsColumnar: Boolean = original.supportsColumnar + + override def doCanonicalize(): FileSourceScanExec = original.doCanonicalize() + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val numOutputRows = longMetric("numOutputRows") + val scanTime = longMetric("scanTime") + inputRDD.asInstanceOf[RDD[ColumnarBatch]].mapPartitionsInternal { + batches => + new Iterator[ColumnarBatch] { + + override def hasNext: Boolean = { + // The `FileScanRDD` returns an iterator which scans the file during the `hasNext` call. + val startNs = System.nanoTime() + val res = batches.hasNext + scanTime += NANOSECONDS.toMillis(System.nanoTime() - startNs) + res + } + + override def next(): ColumnarBatch = { + val batch = batches.next() + numOutputRows += batch.numRows() + batch + } + } + } + } +} diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml index db617112f652..1e4438b84b4f 100644 --- a/gluten-data/pom.xml +++ b/gluten-data/pom.xml @@ -165,6 +165,60 @@ compile + + org.apache.arrow + arrow-dataset + ${arrow.version} + + + io.netty + netty-common + + + io.netty + netty-buffer + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + protobuf-java + com.google.protobuf + + + compile + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + diff --git a/gluten-data/src/main/java/org/apache/gluten/exception/SchemaMismatchException.java b/gluten-data/src/main/java/org/apache/gluten/exception/SchemaMismatchException.java new file mode 100644 index 000000000000..5401a58dcbb8 --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/exception/SchemaMismatchException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.exception; + +public class SchemaMismatchException extends RuntimeException { + public SchemaMismatchException(String var1) { + super(var1); + } +} diff --git a/gluten-data/src/main/java/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorUtils.java b/gluten-data/src/main/java/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorUtils.java new file mode 100644 index 000000000000..7830842148c7 --- /dev/null +++ b/gluten-data/src/main/java/org/apache/spark/sql/execution/vectorized/ArrowColumnVectorUtils.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.vectorized; + +import org.apache.gluten.vectorized.ArrowWritableColumnVector; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.types.CalendarIntervalType; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.DateType; +import org.apache.spark.sql.types.DayTimeIntervalType; +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.types.DecimalType; +import org.apache.spark.sql.types.TimestampNTZType; +import org.apache.spark.sql.types.TimestampType; +import org.apache.spark.sql.types.YearMonthIntervalType; +import org.apache.spark.unsafe.types.CalendarInterval; +import org.apache.spark.unsafe.types.UTF8String; + +import java.math.BigInteger; + +/** + * Utilities to help manipulate data associate with ColumnVectors. These should be used mostly for + * debugging or other non-performance critical paths. These utilities are mostly used to convert + * ColumnVectors into other formats. + */ +public class ArrowColumnVectorUtils { + /** + * Populates the entire `col` with `row[fieldIdx]` This is copied from {@link + * ColumnVectorUtils#populate}. We changed the way to putByteArrays. + */ + public static void populate(WritableColumnVector col, InternalRow row, int fieldIdx) { + ArrowWritableColumnVector arrowCol = (ArrowWritableColumnVector) col; + + int capacity = arrowCol.capacity; + if (row.isNullAt(fieldIdx)) { + arrowCol.putNulls(0, capacity); + } else { + if (arrowCol.dataType() == DataTypes.StringType) { + UTF8String v = row.getUTF8String(fieldIdx); + byte[] bytes = v.getBytes(); + for (int i = 0; i < capacity; i++) { + col.putByteArray(i, bytes); + } + } else { + populatePrimitive(col, row, fieldIdx); + } + } + } + + public static void populatePrimitive(WritableColumnVector col, InternalRow row, int fieldIdx) { + int capacity = col.capacity; + DataType t = col.dataType(); + + if (row.isNullAt(fieldIdx)) { + col.putNulls(0, capacity); + } else { + if (t == DataTypes.BooleanType) { + col.putBooleans(0, capacity, row.getBoolean(fieldIdx)); + } else if (t == DataTypes.BinaryType) { + col.putByteArray(0, row.getBinary(fieldIdx)); + } else if (t == DataTypes.ByteType) { + col.putBytes(0, capacity, row.getByte(fieldIdx)); + } else if (t == DataTypes.ShortType) { + col.putShorts(0, capacity, row.getShort(fieldIdx)); + } else if (t == DataTypes.IntegerType) { + col.putInts(0, capacity, row.getInt(fieldIdx)); + } else if (t == DataTypes.LongType) { + col.putLongs(0, capacity, row.getLong(fieldIdx)); + } else if (t == DataTypes.FloatType) { + col.putFloats(0, capacity, row.getFloat(fieldIdx)); + } else if (t == DataTypes.DoubleType) { + col.putDoubles(0, capacity, row.getDouble(fieldIdx)); + } else if (t == DataTypes.StringType) { + UTF8String v = row.getUTF8String(fieldIdx); + byte[] bytes = v.getBytes(); + for (int i = 0; i < capacity; i++) { + col.putByteArray(i, bytes); + } + } else if (t instanceof DecimalType) { + DecimalType dt = (DecimalType) t; + Decimal d = row.getDecimal(fieldIdx, dt.precision(), dt.scale()); + if (dt.precision() <= Decimal.MAX_INT_DIGITS()) { + col.putInts(0, capacity, (int) d.toUnscaledLong()); + } else if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) { + col.putLongs(0, capacity, d.toUnscaledLong()); + } else { + final BigInteger integer = d.toJavaBigDecimal().unscaledValue(); + byte[] bytes = integer.toByteArray(); + for (int i = 0; i < capacity; i++) { + col.putByteArray(i, bytes, 0, bytes.length); + } + } + } else if (t instanceof CalendarIntervalType) { + CalendarInterval c = (CalendarInterval) row.get(fieldIdx, t); + col.getChild(0).putInts(0, capacity, c.months); + col.getChild(1).putInts(0, capacity, c.days); + col.getChild(2).putLongs(0, capacity, c.microseconds); + } else if (t instanceof DateType || t instanceof YearMonthIntervalType) { + col.putInts(0, capacity, row.getInt(fieldIdx)); + } else if (t instanceof TimestampType + || t instanceof TimestampNTZType + || t instanceof DayTimeIntervalType) { + col.putLongs(0, capacity, row.getLong(fieldIdx)); + } else { + throw new RuntimeException( + String.format( + "DataType %s is not supported" + " in column vectorized reader.", t.sql())); + } + } + } +} diff --git a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala index d43486e61467..4579e015b26e 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala @@ -16,21 +16,34 @@ */ package org.apache.gluten.utils +import org.apache.gluten.exception.SchemaMismatchException +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.execution.vectorized.ArrowColumnVectorUtils +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.utils.{SparkArrowUtil, SparkSchemaUtil} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.arrow.c.{ArrowSchema, CDataDictionaryProvider, Data} +import org.apache.arrow.dataset.file.{FileFormat, FileSystemDatasetFactory} +import org.apache.arrow.dataset.jni.NativeMemoryPool import org.apache.arrow.memory.BufferAllocator +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} +import org.apache.hadoop.fs.FileStatus +import java.net.URI import java.util import scala.collection.JavaConverters._ +import scala.collection.mutable object ArrowUtil extends Logging { @@ -86,4 +99,213 @@ object ArrowUtil extends Logging { } new Schema(fields) } + + def getFormat(format: String): FileFormat = { + format match { + case "parquet" => FileFormat.PARQUET + case "orc" => FileFormat.ORC + case "csv" => FileFormat.CSV + case _ => throw new IllegalArgumentException("Unrecognizable format") + } + } + + def getFormat(format: org.apache.spark.sql.execution.datasources.FileFormat): FileFormat = { + format match { + case _: ParquetFileFormat => + FileFormat.PARQUET + case _: CSVFileFormat => + FileFormat.CSV + case _ => + throw new IllegalArgumentException("Unrecognizable format") + } + } + + private def rewriteUri(encodeUri: String): String = { + val decodedUri = encodeUri + val uri = URI.create(decodedUri) + if (uri.getScheme == "s3" || uri.getScheme == "s3a") { + val s3Rewritten = + new URI("s3", uri.getAuthority, uri.getPath, uri.getQuery, uri.getFragment).toString + return s3Rewritten + } + val sch = uri.getScheme match { + case "hdfs" => "hdfs" + case "file" => "file" + } + val ssp = uri.getScheme match { + case "hdfs" => uri.getSchemeSpecificPart + case "file" => "//" + uri.getSchemeSpecificPart + } + val rewritten = new URI(sch, ssp, uri.getFragment) + rewritten.toString + } + + def makeArrowDiscovery(encodedUri: String, format: FileFormat): FileSystemDatasetFactory = { + val allocator = ArrowBufferAllocators.contextInstance() + val factory = new FileSystemDatasetFactory( + allocator, + NativeMemoryPool.getDefault, // TODO: wait to change + format, + rewriteUri(encodedUri)) + factory + } + + def readSchema(file: FileStatus, format: FileFormat): Option[StructType] = { + val factory: FileSystemDatasetFactory = + makeArrowDiscovery(file.getPath.toString, format) + val schema = factory.inspect() + try { + Option(SparkSchemaUtil.fromArrowSchema(schema)) + } finally { + factory.close() + } + } + + def readSchema(files: Seq[FileStatus], format: FileFormat): Option[StructType] = { + if (files.isEmpty) { + throw new IllegalArgumentException("No input file specified") + } + + readSchema(files.head, format) + } + + def compareStringFunc(caseSensitive: Boolean): (String, String) => Boolean = { + if (caseSensitive) { (str1: String, str2: String) => str1.equals(str2) } + else { (str1: String, str2: String) => str1.equalsIgnoreCase(str2) } + } + + // If user specify schema by .schema(newSchemaDifferentWithFile) + def checkSchema( + requiredField: DataType, + parquetFileFieldType: ArrowType, + parquetFileFields: mutable.Buffer[Field]): Unit = { + val requiredFieldType = + SparkArrowUtil.toArrowType(requiredField, SparkSchemaUtil.getLocalTimezoneID) + if (!requiredFieldType.equals(parquetFileFieldType)) { + val arrowFileSchema = parquetFileFields + .map(f => f.toString) + .reduceLeft((f1, f2) => f1 + "\n" + f2) + throw new SchemaMismatchException( + s"Not support specified schema is different with file schema\n$arrowFileSchema") + } + } + + def getRequestedField( + requiredSchema: StructType, + parquetFileFields: mutable.Buffer[Field], + caseSensitive: Boolean): Schema = { + val compareFunc = compareStringFunc(caseSensitive) + requiredSchema.foreach { + readField => + // TODO: check schema inside of complex type + val matchedFields = + parquetFileFields.filter(field => compareFunc(field.getName, readField.name)) + if (!caseSensitive && matchedFields.size > 1) { + // Need to fail if there is ambiguity, i.e. more than one field is matched + val fieldsString = matchedFields.map(_.getName).mkString("[", ", ", "]") + throw new RuntimeException( + s""" + |Found duplicate field(s) "${readField.name}": $fieldsString + + |in case-insensitive mode""".stripMargin.replaceAll("\n", " ")) + } + if (matchedFields.nonEmpty) { + checkSchema( + readField.dataType, + matchedFields.head.getFieldType.getType, + parquetFileFields) + } + } + + val requestColNames = requiredSchema.map(_.name) + new Schema(parquetFileFields.filter { + field => requestColNames.exists(col => compareFunc(col, field.getName)) + }.asJava) + } + + def loadMissingColumns( + rowCount: Int, + missingSchema: StructType): Array[ArrowWritableColumnVector] = { + + val vectors = + ArrowWritableColumnVector.allocateColumns(rowCount, missingSchema) + vectors.foreach { + vector => + vector.putNulls(0, rowCount) + vector.setValueCount(rowCount) + } + + vectors + } + + def loadPartitionColumns( + rowCount: Int, + partitionSchema: StructType, + partitionValues: InternalRow): Array[ArrowWritableColumnVector] = { + val partitionColumns = ArrowWritableColumnVector.allocateColumns(rowCount, partitionSchema) + (0 until partitionColumns.length).foreach( + i => { + ArrowColumnVectorUtils.populate(partitionColumns(i), partitionValues, i) + partitionColumns(i).setValueCount(rowCount) + partitionColumns(i).setIsConstant() + }) + + partitionColumns + } + + def loadBatch( + input: ArrowRecordBatch, + dataSchema: StructType, + requiredSchema: StructType, + partitionVectors: Array[ArrowWritableColumnVector] = Array.empty, + nullVectors: Array[ArrowWritableColumnVector] = Array.empty): ColumnarBatch = { + val rowCount: Int = input.getLength + + val vectors = + try { + ArrowWritableColumnVector.loadColumns( + rowCount, + SparkSchemaUtil.toArrowSchema(dataSchema), + input, + ArrowBufferAllocators.contextInstance()) + } finally { + input.close() + } + + val totalVectors = if (nullVectors.nonEmpty) { + val finalVectors = + mutable.ArrayBuffer[ArrowWritableColumnVector]() + val requiredIterator = requiredSchema.iterator + val compareFunc = compareStringFunc(SQLConf.get.caseSensitiveAnalysis) + while (requiredIterator.hasNext) { + val field = requiredIterator.next() + finalVectors.append(vectors + .find(vector => compareFunc(vector.getValueVector.getName, field.name)) + .getOrElse { + // The missing column need to be find in nullVectors + val nullVector = + nullVectors.find(vector => compareFunc(vector.getValueVector.getName, field.name)).get + nullVector.setValueCount(rowCount) + nullVector.retain() + nullVector + }) + } + finalVectors.toArray + } else { + vectors + } + + val batch = new ColumnarBatch( + totalVectors.map(_.asInstanceOf[ColumnVector]) ++ + partitionVectors + .map { + vector => + vector.setValueCount(rowCount) + vector.asInstanceOf[ColumnVector] + }, + rowCount + ) + batch + } + } diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderCheckerHelper.scala b/gluten-data/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderCheckerHelper.scala new file mode 100644 index 000000000000..320cad95b904 --- /dev/null +++ b/gluten-data/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderCheckerHelper.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.csv + +import com.univocity.parsers.csv.CsvParser + +object CSVHeaderCheckerHelper { + def checkHeaderColumnNames( + checker: CSVHeaderChecker, + lines: Iterator[String], + tokenizer: CsvParser): Unit = { + checker.checkHeaderColumnNames(lines, tokenizer) + } +} diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkSchemaUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkSchemaUtil.scala index b0b9d2df03a4..b49077bd2740 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkSchemaUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/utils/SparkSchemaUtil.scala @@ -29,6 +29,10 @@ object SparkSchemaUtil { SparkArrowUtil.fromArrowSchema(schema) } + def toArrowSchema(schema: StructType): Schema = { + SparkArrowUtil.toArrowSchema(schema, getLocalTimezoneID) + } + def toArrowSchema(schema: StructType, timeZoneId: String): Schema = { SparkArrowUtil.toArrowSchema(schema, timeZoneId) } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index b9f8c066578a..a319c5ca9897 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -413,10 +413,30 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("column pruning - non-readable file") enableSuite[GlutenCSVv1Suite] .exclude("SPARK-23786: warning should be printed if CSV header doesn't conform to schema") + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] .exclude("SPARK-23786: warning should be printed if CSV header doesn't conform to schema") + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + // file cars.csv include null string, Arrow not support to read + .exclude("old csv data source name works") enableSuite[GlutenCSVLegacyTimeParserSuite] .exclude("SPARK-23786: warning should be printed if CSV header doesn't conform to schema") + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") enableSuite[GlutenJsonV1Suite] // FIXME: Array direct selection fails .exclude("Complex field and type inferring") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 642873028583..d750456b6287 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -202,8 +202,29 @@ class VeloxTestSettings extends BackendTestSettings { // Exception. .exclude("column pruning - non-readable file") enableSuite[GlutenCSVv1Suite] + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] + .exclude("Gluten - test for FAILFAST parsing mode") + // file cars.csv include null string, Arrow not support to read + .exclude("old csv data source name works") enableSuite[GlutenCSVLegacyTimeParserSuite] + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenJsonV1Suite] // FIXME: Array direct selection fails .exclude("Complex field and type inferring") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index a046f0a02a1e..689eaf39ec7a 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -182,8 +182,29 @@ class VeloxTestSettings extends BackendTestSettings { // Exception. .exclude("column pruning - non-readable file") enableSuite[GlutenCSVv1Suite] + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] + .exclude("Gluten - test for FAILFAST parsing mode") + // file cars.csv include null string, Arrow not support to read + .exclude("old csv data source name works") enableSuite[GlutenCSVLegacyTimeParserSuite] + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenJsonV1Suite] // FIXME: Array direct selection fails .exclude("Complex field and type inferring") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 4583396b484d..c2385bd56615 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -185,9 +185,29 @@ class VeloxTestSettings extends BackendTestSettings { // Exception. .exclude("column pruning - non-readable file") enableSuite[GlutenCSVv1Suite] + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] .exclude("Gluten - test for FAILFAST parsing mode") + // file cars.csv include null string, Arrow not support to read + .exclude("old csv data source name works") enableSuite[GlutenCSVLegacyTimeParserSuite] + // file cars.csv include null string, Arrow not support to read + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenJsonV1Suite] // FIXME: Array direct selection fails .exclude("Complex field and type inferring") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala index 1ca70a2cb9a9..7cca2bc3cd7f 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala @@ -16,16 +16,27 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.GlutenConfig + +import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsBaseTrait import org.apache.spark.sql.internal.SQLConf import java.io.File -class GlutenCSVReadSchemaSuite extends CSVReadSchemaSuite with GlutenSQLTestsBaseTrait {} +class GlutenCSVReadSchemaSuite extends CSVReadSchemaSuite with GlutenSQLTestsBaseTrait { -class GlutenHeaderCSVReadSchemaSuite - extends HeaderCSVReadSchemaSuite - with GlutenSQLTestsBaseTrait {} + override def sparkConf: SparkConf = + super.sparkConf + .set(GlutenConfig.NATIVE_ARROW_READER_ENABLED.key, "true") +} + +class GlutenHeaderCSVReadSchemaSuite extends HeaderCSVReadSchemaSuite with GlutenSQLTestsBaseTrait { + + override def sparkConf: SparkConf = + super.sparkConf + .set(GlutenConfig.NATIVE_ARROW_READER_ENABLED.key, "true") +} class GlutenJsonReadSchemaSuite extends JsonReadSchemaSuite with GlutenSQLTestsBaseTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala index 74de5f466268..38e6c9873ee0 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala @@ -16,19 +16,26 @@ */ package org.apache.spark.sql.execution.datasources.csv +import org.apache.gluten.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} +import org.apache.spark.sql.{AnalysisException, GlutenSQLTestsBaseTrait, Row} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DateType, IntegerType, StructType, TimestampType} +import org.apache.spark.sql.types.{DateType, IntegerType, StringType, StructType, TimestampType} import org.scalatest.exceptions.TestFailedException import java.sql.{Date, Timestamp} +import scala.collection.JavaConverters.seqAsJavaListConverter + class GlutenCSVSuite extends CSVSuite with GlutenSQLTestsBaseTrait { + override def sparkConf: SparkConf = + super.sparkConf + .set(GlutenConfig.NATIVE_ARROW_READER_ENABLED.key, "true") + /** Returns full path to the given file in the resource folder */ override protected def testFile(fileName: String): String = { getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + fileName @@ -36,9 +43,68 @@ class GlutenCSVSuite extends CSVSuite with GlutenSQLTestsBaseTrait { } class GlutenCSVv1Suite extends GlutenCSVSuite { + import testImplicits._ override def sparkConf: SparkConf = super.sparkConf .set(SQLConf.USE_V1_SOURCE_LIST, "csv") + + testGluten("SPARK-23786: Ignore column name case if spark.sql.caseSensitive is false") { + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { + withTempPath { + path => + val oschema = new StructType().add("A", StringType) + // change the row content 0 to string bbb in Gluten for test + val odf = spark.createDataFrame(List(Row("bbb")).asJava, oschema) + odf.write.option("header", true).csv(path.getCanonicalPath) + val ischema = new StructType().add("a", StringType) + val idf = spark.read + .schema(ischema) + .option("header", true) + .option("enforceSchema", false) + .csv(path.getCanonicalPath) + checkAnswer(idf, odf) + } + } + } + + testGluten("case sensitivity of filters references") { + Seq(true, false).foreach { + filterPushdown => + withSQLConf(SQLConf.CSV_FILTER_PUSHDOWN_ENABLED.key -> filterPushdown.toString) { + withTempPath { + path => + Seq("""aaa,BBB""", """0,1""", """2,3""") + .toDF() + .repartition(1) + .write + .text(path.getCanonicalPath) + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { + // change the schema to Arrow schema to support read in Gluten + val readback = spark.read + .schema("aaa long, BBB long") + .option("header", true) + .csv(path.getCanonicalPath) + checkAnswer(readback, Seq(Row(2, 3), Row(0, 1))) + checkAnswer(readback.filter($"AAA" === 2 && $"bbb" === 3), Seq(Row(2, 3))) + } + withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { + val readback = spark.read + .schema("aaa long, BBB long") + .option("header", true) + .csv(path.getCanonicalPath) + checkAnswer(readback, Seq(Row(2, 3), Row(0, 1))) + checkError( + exception = intercept[AnalysisException] { + readback.filter($"AAA" === 2 && $"bbb" === 3).collect() + }, + errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", + parameters = Map("objectName" -> "`AAA`", "proposal" -> "`BBB`, `aaa`") + ) + } + } + } + } + } } class GlutenCSVv2Suite extends GlutenCSVSuite { diff --git a/package/pom.xml b/package/pom.xml index 5f5eb0b96fc8..f405124a2a77 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -148,10 +148,11 @@ org.apache.arrow ${gluten.shade.packageName}.org.apache.arrow - + org.apache.arrow.c.* org.apache.arrow.c.jni.* + org.apache.arrow.dataset.** diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 06d72ec57182..7a501e02deb0 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -373,6 +373,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { // Please use `BackendsApiManager.getSettings.enableNativeWriteFiles()` instead def enableNativeWriter: Option[Boolean] = conf.getConf(NATIVE_WRITER_ENABLED) + def enableNativeArrowReader: Boolean = conf.getConf(NATIVE_ARROW_READER_ENABLED) + def directorySizeGuess: Long = conf.getConf(DIRECTORY_SIZE_GUESS) def filePreloadThreshold: Long = @@ -1470,6 +1472,13 @@ object GlutenConfig { .booleanConf .createOptional + val NATIVE_ARROW_READER_ENABLED = + buildConf("spark.gluten.sql.native.arrow.reader.enabled") + .internal() + .doc("This is config to specify whether to enable the native columnar csv reader") + .booleanConf + .createWithDefault(false) + val NATIVE_WRITE_FILES_COLUMN_METADATA_EXCLUSION_LIST = buildConf("spark.gluten.sql.native.writeColumnMetadataExclusionList") .doc( diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala index 0895c7e9a5d0..dbefc22ef7a2 100644 --- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala @@ -26,6 +26,7 @@ import org.apache.spark.shuffle.ShuffleHandle import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions.{Attribute, BinaryExpression, Expression} import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -229,4 +230,6 @@ trait SparkShims { def withAnsiEvalMode(expr: Expression): Boolean = false + def dateTimestampFormatInReadIsDefaultValue(csvOptions: CSVOptions, timeZone: String): Boolean + } diff --git a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala index 4d5488de6037..97251a7ef386 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala @@ -26,11 +26,13 @@ import org.apache.spark.shuffle.ShuffleHandle import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, BinaryExpression, Expression} import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{Distribution, HashClusteredDistribution} import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.{FileSourceScanExec, PartitionedFileUtil, SparkPlan} @@ -226,4 +228,12 @@ class Spark32Shims extends SparkShims { override def getCommonPartitionValues(batchScan: BatchScanExec): Option[Seq[(InternalRow, Int)]] = null + + override def dateTimestampFormatInReadIsDefaultValue( + csvOptions: CSVOptions, + timeZone: String): Boolean = { + val default = new CSVOptions(CaseInsensitiveMap(Map()), csvOptions.columnPruning, timeZone) + csvOptions.dateFormat == default.dateFormat && + csvOptions.timestampFormat == default.timestampFormat + } } diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 2c9e952cd841..6378af3ba023 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -18,7 +18,8 @@ package org.apache.spark.sql.execution import org.apache.gluten.metrics.GlutenTimeMetric -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, BoundReference, DynamicPruningExpression, Expression, PlanExpression, Predicate} import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} @@ -154,3 +155,18 @@ abstract class FileSourceScanExecShim( selected } } + +abstract class ArrowFileSourceScanLikeShim(original: FileSourceScanExec) + extends DataSourceScanExec { + override val nodeNamePrefix: String = "ArrowFile" + + override lazy val metrics = original.metrics + + override def tableIdentifier: Option[TableIdentifier] = original.tableIdentifier + + override def inputRDDs(): Seq[RDD[InternalRow]] = original.inputRDDs() + + override def relation: HadoopFsRelation = original.relation + + override protected def metadata: Map[String, String] = original.metadata +} diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala index d264bd1acc55..58dc0a00cec4 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala @@ -27,11 +27,13 @@ import org.apache.spark.shuffle.ShuffleHandle import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{BloomFilterAggregate, RegrR2, TypedImperativeAggregate} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution} import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.catalyst.util.TimestampFormatter import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform @@ -316,4 +318,13 @@ class Spark33Shims extends SparkShims { } override def supportsRowBased(plan: SparkPlan): Boolean = plan.supportsRowBased + + override def dateTimestampFormatInReadIsDefaultValue( + csvOptions: CSVOptions, + timeZone: String): Boolean = { + val default = new CSVOptions(CaseInsensitiveMap(Map()), csvOptions.columnPruning, timeZone) + csvOptions.dateFormatInRead == default.dateFormatInRead && + csvOptions.timestampFormatInRead == default.timestampFormatInRead && + csvOptions.timestampNTZFormatInRead == default.timestampNTZFormatInRead + } } diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index f0beb19cd557..5d3f6275cd9e 100644 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -18,7 +18,8 @@ package org.apache.spark.sql.execution import org.apache.gluten.metrics.GlutenTimeMetric -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, BoundReference, DynamicPruningExpression, Expression, FileSourceMetadataAttribute, PlanExpression, Predicate} import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation, PartitionDirectory} import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils @@ -161,3 +162,18 @@ abstract class FileSourceScanExecShim( selected } } + +abstract class ArrowFileSourceScanLikeShim(original: FileSourceScanExec) + extends DataSourceScanExec { + override val nodeNamePrefix: String = "ArrowFile" + + override lazy val metrics = original.metrics + + override def tableIdentifier: Option[TableIdentifier] = original.tableIdentifier + + override def inputRDDs(): Seq[RDD[InternalRow]] = original.inputRDDs() + + override def relation: HadoopFsRelation = original.relation + + override protected def metadata: Map[String, String] = original.metadata +} diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 53d7d7f8f8b2..0b045972a8b9 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -29,12 +29,13 @@ import org.apache.spark.shuffle.ShuffleHandle import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution, KeyGroupedPartitioning, Partitioning} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.{InternalRowComparableWrapper, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, InternalRowComparableWrapper, TimestampFormatter} import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} @@ -410,4 +411,13 @@ class Spark34Shims extends SparkShims { case _ => false } } + + override def dateTimestampFormatInReadIsDefaultValue( + csvOptions: CSVOptions, + timeZone: String): Boolean = { + val default = new CSVOptions(CaseInsensitiveMap(Map()), csvOptions.columnPruning, timeZone) + csvOptions.dateFormatInRead == default.dateFormatInRead && + csvOptions.timestampFormatInRead == default.timestampFormatInRead && + csvOptions.timestampNTZFormatInRead == default.timestampNTZFormatInRead + } } diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 3c2282358173..4fc09f3aef35 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -18,7 +18,8 @@ package org.apache.spark.sql.execution import org.apache.gluten.metrics.GlutenTimeMetric -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, BoundReference, Expression, FileSourceConstantMetadataAttribute, FileSourceGeneratedMetadataAttribute, FileSourceMetadataAttribute, PlanExpression, Predicate} import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation, PartitionDirectory} import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils @@ -121,3 +122,26 @@ abstract class FileSourceScanExecShim( selected } } + +abstract class ArrowFileSourceScanLikeShim(original: FileSourceScanExec) + extends FileSourceScanLike { + override val nodeNamePrefix: String = "ArrowFile" + + override def tableIdentifier: Option[TableIdentifier] = original.tableIdentifier + + override def inputRDDs(): Seq[RDD[InternalRow]] = original.inputRDDs() + + override def dataFilters: Seq[Expression] = original.dataFilters + + override def disableBucketedScan: Boolean = original.disableBucketedScan + + override def optionalBucketSet: Option[BitSet] = original.optionalBucketSet + + override def optionalNumCoalescedBuckets: Option[Int] = original.optionalNumCoalescedBuckets + + override def partitionFilters: Seq[Expression] = original.partitionFilters + + override def relation: HadoopFsRelation = original.relation + + override def requiredSchema: StructType = original.requiredSchema +} diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index f78d4ca6b52d..c839c8c2af03 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -28,13 +28,14 @@ import org.apache.spark.shuffle.ShuffleHandle import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, InternalRow} import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{BloomFilterAggregate, RegrIntercept, RegrR2, RegrReplacement, RegrSlope, RegrSXY, TypedImperativeAggregate} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution, KeyGroupedPartitioning, Partitioning} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.types.DataTypeUtils -import org.apache.spark.sql.catalyst.util.{InternalRowComparableWrapper, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, InternalRowComparableWrapper, TimestampFormatter} import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} @@ -438,4 +439,13 @@ class Spark35Shims extends SparkShims { case _ => false } } + + override def dateTimestampFormatInReadIsDefaultValue( + csvOptions: CSVOptions, + timeZone: String): Boolean = { + val default = new CSVOptions(CaseInsensitiveMap(Map()), csvOptions.columnPruning, timeZone) + csvOptions.dateFormatInRead == default.dateFormatInRead && + csvOptions.timestampFormatInRead == default.timestampFormatInRead && + csvOptions.timestampNTZFormatInRead == default.timestampNTZFormatInRead + } } diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 77786ba0d70e..b3bb2d2934e3 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -18,7 +18,8 @@ package org.apache.spark.sql.execution import org.apache.gluten.metrics.GlutenTimeMetric -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, BoundReference, DynamicPruningExpression, Expression, FileSourceConstantMetadataAttribute, FileSourceGeneratedMetadataAttribute, FileSourceMetadataAttribute, PlanExpression, Predicate} import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation, PartitionDirectory} import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetUtils} @@ -124,3 +125,26 @@ abstract class FileSourceScanExecShim( selected } } + +abstract class ArrowFileSourceScanLikeShim(original: FileSourceScanExec) + extends FileSourceScanLike { + override val nodeNamePrefix: String = "ArrowFile" + + override def tableIdentifier: Option[TableIdentifier] = original.tableIdentifier + + override def inputRDDs(): Seq[RDD[InternalRow]] = original.inputRDDs() + + override def dataFilters: Seq[Expression] = original.dataFilters + + override def disableBucketedScan: Boolean = original.disableBucketedScan + + override def optionalBucketSet: Option[BitSet] = original.optionalBucketSet + + override def optionalNumCoalescedBuckets: Option[Int] = original.optionalNumCoalescedBuckets + + override def partitionFilters: Seq[Expression] = original.partitionFilters + + override def relation: HadoopFsRelation = original.relation + + override def requiredSchema: StructType = original.requiredSchema +} From 233850c625dd2274620a363214c2a694a1c09a10 Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Wed, 8 May 2024 20:41:23 +0800 Subject: [PATCH 028/402] [GLUTEN-5651][CH] Fix error 'Illegal type of argument of function parseDateTimeInJodaSyntaxOrNull, expected String, got Date32' when executing to_date/to_timestamp (#5652) Fix error 'Illegal type of argument of function parseDateTimeInJodaSyntaxOrNull, expected String, got Date32' when executing to_date/to_timestamp. RC: the spark function `to_date/to_timestamp` are mapping to the CH function `parseDateTimeInJodaSyntaxOrNull` when they execute with the specified format, but the CH function `parseDateTimeInJodaSyntaxOrNull` can not support the data type `DateType or TimestampType` as the input data type, and spark supports. Close #5651. --- .../GlutenClickHouseTPCHNullableSuite.scala | 10 ++++- ...enClickHouseTPCHSaltNullParquetSuite.scala | 13 ++++-- .../RewriteToDateExpresstionRule.scala | 42 +++++++++++++++++-- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala index fe6afedf4d5a..0eb4de74209b 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala @@ -19,6 +19,7 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig import org.apache.spark.SparkConf +import org.apache.spark.sql.catalyst.expressions.Alias import org.apache.spark.sql.catalyst.optimizer.BuildLeft class GlutenClickHouseTPCHNullableSuite extends GlutenClickHouseTPCHAbstractSuite { @@ -235,7 +236,14 @@ class GlutenClickHouseTPCHNullableSuite extends GlutenClickHouseTPCHAbstractSuit case project: ProjectExecTransformer => project } assert(project.size == 1) - assert(project.apply(0).projectList.toString().contains("from_unixtime") == conf._2) + assert( + project + .apply(0) + .projectList(0) + .asInstanceOf[Alias] + .child + .toString() + .contains("from_unixtime") == conf._2) }) } }) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 20638615d3c8..a1bba300ed22 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -49,8 +49,6 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr .set("spark.sql.shuffle.partitions", "5") .set("spark.sql.autoBroadcastJoinThreshold", "10MB") .set("spark.gluten.supported.scala.udfs", "my_add") -// .set("spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level", "trace") -// .set("spark.sql.planChangeLog.level", "error") } override protected val createNullableTables = true @@ -1271,8 +1269,15 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr } test("test 'to_date/to_timestamp'") { - val sql = "select to_date(concat('2022-01-0', cast(id+1 as String)), 'yyyy-MM-dd')," + - "to_timestamp(concat('2022-01-01 10:30:0', cast(id+1 as String)), 'yyyy-MM-dd HH:mm:ss') " + + val sql = "select to_date(concat('2022-01-0', cast(id+1 as String)), 'yyyy-MM-dd') as a1," + + "to_timestamp(concat('2022-01-01 10:30:0', cast(id+1 as String)), 'yyyy-MM-dd HH:mm:ss') as a2," + + "to_date(date_add(date'2024-05-07', cast(id as int)), 'yyyy-MM-dd') as a3, " + + "to_date(date_add(date'2024-05-07', cast(id as int)), 'yyyyMMdd') as a4, " + + "to_date(date_add(date'2024-05-07', cast(id as int)), 'yyyy-MM') as a5, " + + "to_date(date_add(date'2024-05-07', cast(id as int)), 'yyyy') as a6, " + + "to_date(to_timestamp(concat('2022-01-01 10:30:0', cast(id+1 as String))), 'yyyy-MM-dd HH:mm:ss') as a7, " + + "to_timestamp(date_add(date'2024-05-07', cast(id as int)), 'yyyy-MM') as a8, " + + "to_timestamp(to_timestamp(concat('2022-01-01 10:30:0', cast(id+1 as String))), 'yyyy-MM-dd HH:mm:ss') as a9 " + "from range(9)" runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer]) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala index f809bb70f971..34d162d71f5f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala @@ -64,10 +64,25 @@ class RewriteToDateExpresstionRule(session: SparkSession, conf: SQLConf) } private def visitExpression(expression: NamedExpression): NamedExpression = expression match { - case Alias(c, _) if c.isInstanceOf[ParseToDate] => + case a @ Alias(c, _) if c.isInstanceOf[ParseToDate] => val newToDate = rewriteParseToDate(c.asInstanceOf[ParseToDate]) if (!newToDate.fastEquals(c)) { - Alias(newToDate, newToDate.toString())() + a.copy(newToDate, a.name)( + a.exprId, + a.qualifier, + a.explicitMetadata, + a.nonInheritableMetadataKeys) + } else { + expression + } + case a @ Alias(c, _) if c.isInstanceOf[ParseToTimestamp] => + val newToTimestamp = rewriteParseToTimestamp(c.asInstanceOf[ParseToTimestamp]) + if (!newToTimestamp.fastEquals(c)) { + a.copy(newToTimestamp, a.name)( + a.exprId, + a.qualifier, + a.explicitMetadata, + a.nonInheritableMetadataKeys) } else { expression } @@ -81,11 +96,32 @@ class RewriteToDateExpresstionRule(session: SparkSession, conf: SQLConf) val unixTimestamp = fromUnixTime.left.asInstanceOf[UnixTimestamp] val newLeft = unixTimestamp.left new ParseToDate(newLeft) + case date: Expression + if date.dataType.isInstanceOf[DateType] || date.dataType.isInstanceOf[TimestampType] => + // When the data type of the left child in the ParseToDate is the DateType or TimestampType, + // it will not deal with the format, + // also CH backend can not support the DateType or TimestampType as input data type + Cast(date, toDate.dataType, Some(SQLConf.get.sessionLocalTimeZone)) case _ => toDate } + private def rewriteParseToTimestamp(toTimestamp: ParseToTimestamp): Expression = + toTimestamp.left match { + case timestamp: Expression + if (timestamp.dataType.isInstanceOf[DateType] || + timestamp.dataType.isInstanceOf[TimestampType]) => + // When the data type of the left child in the ParseToDate is the DateType or TimestampType, + // it will not deal with the format, + // also CH backend can not support the DateType or TimestampType as input data type + Cast(timestamp, toTimestamp.dataType, Some(SQLConf.get.sessionLocalTimeZone)) + case _ => toTimestamp + } + private def canRewrite(project: Project): Boolean = { project.projectList.exists( - expr => expr.isInstanceOf[Alias] && expr.asInstanceOf[Alias].child.isInstanceOf[ParseToDate]) + expr => + expr.isInstanceOf[Alias] && + (expr.asInstanceOf[Alias].child.isInstanceOf[ParseToDate] || + expr.asInstanceOf[Alias].child.isInstanceOf[ParseToTimestamp])) } } From b2bb2fbafbe370d2b25689a8dd36cfb6074ff2d1 Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Wed, 8 May 2024 20:57:13 +0800 Subject: [PATCH 029/402] [VL] Generate hdfs-client.xml for libhdfs (#5661) * [VL] Generate hdfs-client.xml for libhdfs * style --------- Co-authored-by: Kent Yao --- .../backendsapi/velox/VeloxBackend.scala | 2 + .../org/apache/gluten/GlutenPlugin.scala | 5 +- .../backendsapi/BackendSettingsApi.scala | 2 + .../org/apache/spark/HdfsConfGenerator.scala | 94 +++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 0d38fd07c570..aad8ff5d5d55 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -510,4 +510,6 @@ object VeloxBackendSettings extends BackendSettingsApi { override def shouldRewriteCollect(): Boolean = true override def supportColumnarArrowUdf(): Boolean = true + + override def generateHdfsConfForLibhdfs(): Boolean = true } diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index b045e4034a1a..adb3f418907f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -26,7 +26,7 @@ import org.apache.gluten.extension.{ColumnarOverrides, OthersExtensionOverrides, import org.apache.gluten.test.TestStats import org.apache.gluten.utils.TaskListener -import org.apache.spark.{SparkConf, SparkContext, TaskFailedReason} +import org.apache.spark.{HdfsConfGenerator, SparkConf, SparkContext, TaskFailedReason} import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin} import org.apache.spark.internal.Logging import org.apache.spark.listener.GlutenListenerFactory @@ -68,6 +68,9 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { } setPredefinedConfigs(sc, conf) + if (BackendsApiManager.getSettings.generateHdfsConfForLibhdfs()) { + HdfsConfGenerator.addHdfsClientToSparkWorkDirectory(sc) + } // Initialize Backends API BackendsApiManager.initialize() BackendsApiManager.getListenerApiInstance.onDriverStart(conf) diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index e02c2290ae59..ac8c2a436f83 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -154,4 +154,6 @@ trait BackendSettingsApi { def shouldRewriteCollect(): Boolean = false def supportColumnarArrowUdf(): Boolean = false + + def generateHdfsConfForLibhdfs(): Boolean = false } diff --git a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala new file mode 100644 index 000000000000..9756837d96e5 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark + +import org.apache.spark.internal.Logging +import org.apache.spark.util.Utils + +import org.apache.hadoop.conf.Configuration + +import java.io.{File, FileOutputStream} + +import scala.collection.JavaConverters._ + +object HdfsConfGenerator extends Logging { + private val addFileMethod = { + val m = classOf[SparkContext].getDeclaredMethod( + "addFile", + classOf[String], + classOf[Boolean], + classOf[Boolean], + classOf[Boolean]) + m.setAccessible(true) + m + } + + private def addFileOnSubmit(sc: SparkContext, path: String): Unit = { + addFileMethod.invoke(sc, path, Boolean.box(false), Boolean.box(true), Boolean.box(false)) + // Overwrite the spark internal config `spark.app.initial.file.urls`, + // so that the file can be available before initializing executor plugin. + assert(sc.addedFiles.nonEmpty) + sc.conf.set("spark.app.initial.file.urls", sc.addedFiles.keys.toSeq.mkString(",")) + } + + private def ignoreKey(key: String): Boolean = { + key.startsWith("yarn.") || key.startsWith("mapreduce.") || key.startsWith("hive.") || key + .startsWith("io.") || key.startsWith("ipc.") + } + + private def hasLibHdfsConf(sc: SparkContext): Boolean = { + sc.conf.getExecutorEnv.toMap.contains("LIBHDFS3_CONF") || + sc.listFiles().exists(_.contains("hdfs-client.xml")) || + sys.env + .get("SPARK_CONF_DIR") + .orElse(sys.env.get("SPARK_HOME").map(t => s"$t${File.separator}conf")) + .exists(conf => new File(s"$conf${File.separator}hdfs-client.xml").isFile) + } + + def addHdfsClientToSparkWorkDirectory(sc: SparkContext): Unit = { + // Only generate hdfs-client.xml in hdfs env + if (sc.hadoopConfiguration.get("dfs.nameservices") == null) { + return + } + + // Do nothing if people have set config + if (hasLibHdfsConf(sc)) { + return + } + + val tmp = Utils.createTempDir() + // scalastyle:off + // See https://github.com/apache/hawq/blob/e9d43144f7e947e071bba48871af9da354d177d0/src/backend/utils/misc/etc/hdfs-client.xml + // scalastyle:on + val hdfsClientConfFile = new File(tmp, "hdfs-client.xml") + val output = new FileOutputStream(hdfsClientConfFile) + try { + val config = new Configuration(false) + sc.hadoopConfiguration + .iterator() + .asScala + .filterNot(x => ignoreKey(x.getKey)) + .foreach(x => config.set(x.getKey, x.getValue)) + config.writeXml(output) + val tmpPath = hdfsClientConfFile.getAbsolutePath + addFileOnSubmit(sc, tmpPath) + logInfo(s"Added hdfs-client.xml for libhdfs, tmp path: $tmpPath.") + } finally { + output.close() + } + } +} From 82e50ab196caff398013a3e76ca3b854a1156243 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Wed, 8 May 2024 21:52:49 +0800 Subject: [PATCH 030/402] [GLUTEN-5639] [CH] Support spark.sql.decimalOperations.allowPrecisionLoss = true (#5640) --- .../backendsapi/clickhouse/CHBackend.scala | 1 - .../GlutenClickHouseDecimalSuite.scala | 66 ++++--- .../GlutenClickHouseHiveTableSuite.scala | 25 +-- .../GlutenClickHouseTPCDSAbstractSuite.scala | 1 - .../GlutenClickHouseTPCHAbstractSuite.scala | 1 - .../Parser/SerializedPlanParser.cpp | 3 +- .../scalar_function_parser/arithmetic.cpp | 179 ++---------------- .../gluten/utils/DecimalArithmeticUtil.scala | 4 +- .../apache/spark/sql/GlutenTestsTrait.scala | 1 - 9 files changed, 73 insertions(+), 208 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index da6c60d8aea1..bc0c8d1c07f2 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -256,7 +256,6 @@ object CHBackendSettings extends BackendSettingsApi with Logging { override def shuffleSupportedCodec(): Set[String] = GLUTEN_CLICKHOUSE_SHUFFLE_SUPPORTED_CODEC override def needOutputSchemaForPlan(): Boolean = true - override def allowDecimalArithmetic: Boolean = !SQLConf.get.decimalOperationsAllowPrecisionLoss override def transformCheckOverflow: Boolean = false override def requiredInputFilePaths(): Boolean = true diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala index e5d827e26d99..c41ea0ccb2ea 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala @@ -20,6 +20,7 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.{col, rand, when} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import java.io.File @@ -53,7 +54,6 @@ class GlutenClickHouseDecimalSuite .set("spark.io.compression.codec", "snappy") .set("spark.sql.shuffle.partitions", "5") .set("spark.sql.autoBroadcastJoinThreshold", "10MB") - .set("spark.sql.decimalOperations.allowPrecisionLoss", "false") } override def beforeAll(): Unit = { @@ -300,35 +300,45 @@ class GlutenClickHouseDecimalSuite customCheck = customCheck, noFallBack = noFallBack) } - - Range - .inclusive(1, 22) - .foreach( - sql_num => { - decimalTPCHTables.foreach( - dt => { - val decimalType = dt._1 - test(s"TPCH Decimal(${decimalType.precision},${decimalType.scale}) Q$sql_num") { - var noFallBack = true - var compareResult = true - if (sql_num == 16 || sql_num == 21) { - noFallBack = false - } - - if (dt._2.contains(sql_num)) { - compareResult = false + Seq("true", "false").foreach { + allowPrecisionLoss => + Range + .inclusive(1, 22) + .foreach { + sql_num => + { + decimalTPCHTables.foreach { + dt => + { + val decimalType = dt._1 + test(s"""TPCH Decimal(${decimalType.precision},${decimalType.scale}) + | Q$sql_num[allowPrecisionLoss=$allowPrecisionLoss]""".stripMargin) { + var noFallBack = true + var compareResult = true + if (sql_num == 16 || sql_num == 21) { + noFallBack = false + } + + if (dt._2.contains(sql_num)) { + compareResult = false + } + + spark.sql(s"use decimal_${decimalType.precision}_${decimalType.scale}") + withSQLConf( + (SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key, allowPrecisionLoss)) { + runTPCHQuery( + sql_num, + tpchQueries, + compareResult = compareResult, + noFallBack = noFallBack) { _ => {} } + } + spark.sql(s"use default") + } + } } - - spark.sql(s"use decimal_${decimalType.precision}_${decimalType.scale}") - runTPCHQuery( - sql_num, - tpchQueries, - compareResult = compareResult, - noFallBack = noFallBack) { _ => {} } - spark.sql(s"use default") } - }) - }) + } + } test("fix decimal precision overflow") { val sql = diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala index 0ac64ca443c5..3c993b622018 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala @@ -1235,17 +1235,20 @@ class GlutenClickHouseHiveTableSuite } test("GLUTEN-3452: Bug fix decimal divide") { - withSQLConf((SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key, "false")) { - val table_create_sql = - """ - | create table test_tbl_3452(d1 decimal(12,2), d2 decimal(15,3)) stored as parquet; - |""".stripMargin - val data_insert_sql = "insert into test_tbl_3452 values(13.0, 0),(11, NULL), (12.3, 200)" - val select_sql = "select d1/d2, d1/0, d1/cast(0 as decimal) from test_tbl_3452" - spark.sql(table_create_sql); - spark.sql(data_insert_sql) - compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) - spark.sql("drop table test_tbl_3452") + val table_create_sql = + """ + | create table test_tbl_3452(d1 decimal(12,2), d2 decimal(15,3)) stored as parquet; + |""".stripMargin + val data_insert_sql = "insert into test_tbl_3452 values(13.0, 0),(11, NULL), (12.3, 200)" + spark.sql(table_create_sql) + spark.sql(data_insert_sql) + Seq("true", "false").foreach { + s => + withSQLConf((SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key, s)) { + val select_sql = "select d1/d2, d1/0, d1/cast(0 as decimal) from test_tbl_3452" + compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) + } } + spark.sql("drop table test_tbl_3452") } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala index a85593c07ab7..bcdc1f5ef514 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala @@ -158,7 +158,6 @@ abstract class GlutenClickHouseTPCDSAbstractSuite .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.sql.warehouse.dir", warehouse) - .set("spark.sql.decimalOperations.allowPrecisionLoss", "false") /* .set("spark.sql.catalogImplementation", "hive") .set("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=${ metaStorePathAbsolute + "/metastore_db"};create=true") */ diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala index b65b6c35be25..8d671e29f18b 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala @@ -573,7 +573,6 @@ abstract class GlutenClickHouseTPCHAbstractSuite .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.sql.warehouse.dir", warehouse) - .set("spark.sql.decimalOperations.allowPrecisionLoss", "false") /* .set("spark.sql.catalogImplementation", "hive") .set("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=${ metaStorePathAbsolute + "/metastore_db"};create=true") */ diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 82acba37f7d8..a26f78699dc8 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -929,7 +929,8 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( } } - if (function_signature.find("check_overflow:", 0) != function_signature.npos) + /// TODO: FunctionParser for check_overflow and make_decimal + if (function_signature.find("check_overflow:", 0) != String::npos) { if (scalar_function.arguments().size() < 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "check_overflow function requires at least two args."); diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp index ec056da45e07..2a6e435667aa 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp @@ -45,122 +45,12 @@ class DecimalType Int32 scale; private: - static DecimalType bounded_to_spark(const Int32 precision, const Int32 scale) - { - return DecimalType(std::min(precision, spark_max_precision), std::min(scale, spark_max_scale)); - } static DecimalType bounded_to_click_house(const Int32 precision, const Int32 scale) { return DecimalType(std::min(precision, chickhouse_max_precision), std::min(scale, chickhouse_max_scale)); } - static void check_negative_scale(const Int32 scale) - { - /// only support spark.sql.legacy.allowNegativeScaleOfDecimal == false - if (scale < 0) - throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Negative scale is not supported"); - } - - static DecimalType adjust_precision_scale(const Int32 precision, const Int32 scale) - { - check_negative_scale(scale); - assert(precision >= scale); - - if (precision <= spark_max_precision) - { - // Adjustment only needed when we exceed max precision - return DecimalType(precision, scale); - } - else if (scale < 0) - { - // Decimal can have negative scale (SPARK-24468). In this case, we cannot allow a precision - // loss since we would cause a loss of digits in the integer part. - // In this case, we are likely to meet an overflow. - return DecimalType(spark_max_precision, scale); - } - else - { - // Precision/scale exceed maximum precision. Result must be adjusted to MAX_PRECISION. - const int intDigits = precision - scale; - - // If original scale is less than MINIMUM_ADJUSTED_SCALE, use original scale value; otherwise - // preserve at least MINIMUM_ADJUSTED_SCALE fractional digits - const int minScaleValue = std::min(scale, minimum_adjusted_scale); - - // The resulting scale is the maximum between what is available without causing a loss of - // digits for the integer part of the decimal and the minimum guaranteed scale, which is - // computed above - const int adjusted_scale = std::max(spark_max_precision - intDigits, minScaleValue); - return DecimalType(spark_max_precision, adjusted_scale); - } - } public: - /// The formula follows Hive which is based on the SQL standard and MS SQL: - /// https://cwiki.apache.org/confluence/download/attachments/27362075/Hive_Decimal_Precision_Scale_Support.pdf - /// https://msdn.microsoft.com/en-us/library/ms190476.aspx - /// Result Precision: max(s1, s2) + max(p1-s1, p2-s2) + 1 - /// Result Scale: max(s1, s2) - /// +, - - static DecimalType - resultAddSubstractDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2, bool allowPrecisionLoss = true) - { - const Int32 scale = std::max(s1, s2); - const Int32 precision = std::max(p1 - s1, p2 - s2) + scale + 1; - - if (allowPrecisionLoss) - return adjust_precision_scale(precision, scale); - else - return bounded_to_spark(precision, scale); - } - - /// The formula follows Hive which is based on the SQL standard and MS SQL: - /// https://cwiki.apache.org/confluence/download/attachments/27362075/Hive_Decimal_Precision_Scale_Support.pdf - /// https://msdn.microsoft.com/en-us/library/ms190476.aspx - /// Result Precision: p1 - s1 + s2 + max(6, s1 + p2 + 1) - /// Result Scale: max(6, s1 + p2 + 1) - static DecimalType - resultDivideDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2, bool allowPrecisionLoss = true) - { - if (allowPrecisionLoss) - { - const Int32 Int32Dig = p1 - s1 + s2; - const Int32 scale = std::max(minimum_adjusted_scale, s1 + p2 + 1); - const Int32 prec = Int32Dig + scale; - return adjust_precision_scale(prec, scale); - } - else - { - Int32 Int32Dig = std::min(spark_max_scale, p1 - s1 + s2); - Int32 decDig = std::min(spark_max_scale, std::max(minimum_adjusted_scale, s1 + p2 + 1)); - Int32 diff = (Int32Dig + decDig) - spark_max_scale; - - if (diff > 0) - { - decDig -= diff / 2 + 1; - Int32Dig = spark_max_scale - decDig; - } - - return bounded_to_spark(Int32Dig + decDig, decDig); - } - } - - /// The formula follows Hive which is based on the SQL standard and MS SQL: - /// https://cwiki.apache.org/confluence/download/attachments/27362075/Hive_Decimal_Precision_Scale_Support.pdf - /// https://msdn.microsoft.com/en-us/library/ms190476.aspx - /// Result Precision: p1 + p2 + 1 - /// Result Scale: s1 + s2 - static DecimalType - resultMultiplyDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2, bool allowPrecisionLoss = true) - { - const Int32 scale = s1 + s2; - const Int32 precision = p1 + p2 + 1; - - if (allowPrecisionLoss) - return adjust_precision_scale(precision, scale); - else - return bounded_to_spark(precision, scale); - } - static DecimalType evalAddSubstractDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) { const Int32 scale = s1; @@ -215,48 +105,28 @@ class FunctionParserBinaryArithmetic : public FunctionParser return new_args; } - DecimalType getDecimalType(const DataTypePtr & left, const DataTypePtr & right, const bool resultType) const + DecimalType getDecimalType(const DataTypePtr & left, const DataTypePtr & right) const { assert(isDecimal(left) && isDecimal(right)); const Int32 p1 = getDecimalPrecision(*left); const Int32 s1 = getDecimalScale(*left); const Int32 p2 = getDecimalPrecision(*right); const Int32 s2 = getDecimalScale(*right); - return resultType ? internalResultType(p1, s1, p2, s2) : internalEvalType(p1, s1, p2, s2); + return internalEvalType(p1, s1, p2, s2); } - virtual DecimalType internalResultType(Int32 p1, Int32 s1, Int32 p2, Int32 s2) const = 0; virtual DecimalType internalEvalType(Int32 p1, Int32 s1, Int32 p2, Int32 s2) const = 0; const ActionsDAG::Node * checkDecimalOverflow(ActionsDAGPtr & actions_dag, const ActionsDAG::Node * func_node, Int32 precision, Int32 scale) const { + //TODO: checkDecimalOverflowSpark throw exception per configuration const DB::ActionsDAG::NodeRawConstPtrs overflow_args = {func_node, plan_parser->addColumn(actions_dag, std::make_shared(), precision), plan_parser->addColumn(actions_dag, std::make_shared(), scale)}; return toFunctionNode(actions_dag, "checkDecimalOverflowSparkOrNull", overflow_args); } - const DB::ActionsDAG::Node * convertNodeTypeIfNeeded( - const substrait::Expression_ScalarFunction & substrait_func, - const DB::ActionsDAG::Node * func_node, - DB::ActionsDAGPtr & actions_dag) const override - { - const auto & substrait_type = substrait_func.output_type(); - if (const auto result_type = TypeParser::parseType(substrait_type); isDecimalOrNullableDecimal(result_type)) - { - const auto a = removeNullable(result_type); - const auto b = removeNullable(func_node->result_type); - if (a->equals(*b)) - return func_node; - - // as stated in isTypeMatched， currently we don't change nullability of the result type - const std::string type_name = func_node->result_type->isNullable() ? wrapNullableType(true, result_type)->getName() - : removeNullable(result_type)->getName(); - return ActionsDAGUtil::convertNodeType(actions_dag, func_node, type_name, func_node->result_name, DB::CastType::accurateOrNull); - } - return FunctionParser::convertNodeTypeIfNeeded(substrait_func, func_node, actions_dag); - } virtual const DB::ActionsDAG::Node * createFunctionNode(DB::ActionsDAGPtr & actions_dag, const String & func_name, const DB::ActionsDAG::NodeRawConstPtrs & args) const @@ -280,7 +150,7 @@ class FunctionParserBinaryArithmetic : public FunctionParser if (converted) { - const DecimalType evalType = getDecimalType(left_type, right_type, false); + const DecimalType evalType = getDecimalType(left_type, right_type); parsed_args = convertBinaryArithmeticFunDecimalArgs(actions_dag, parsed_args, evalType, substrait_func); } @@ -288,17 +158,20 @@ class FunctionParserBinaryArithmetic : public FunctionParser if (converted) { - const auto parsed_outputType = removeNullable(TypeParser::parseType(substrait_func.output_type())); - assert(isDecimal(parsed_outputType)); - const Int32 parsed_precision = getDecimalPrecision(*parsed_outputType); - const Int32 parsed_scale = getDecimalScale(*parsed_outputType); - + const auto parsed_output_type = removeNullable(TypeParser::parseType(substrait_func.output_type())); + assert(isDecimal(parsed_output_type)); + const Int32 parsed_precision = getDecimalPrecision(*parsed_output_type); + const Int32 parsed_scale = getDecimalScale(*parsed_output_type); + func_node = checkDecimalOverflow(actions_dag, func_node, parsed_precision, parsed_scale); #ifndef NDEBUG - const auto [precision, scale] = getDecimalType(left_type, right_type, true); - // assert(parsed_precision == precision); - // assert(parsed_scale == scale); + const auto output_type = removeNullable(func_node->result_type); + const Int32 output_precision = getDecimalPrecision(*output_type); + const Int32 output_scale = getDecimalScale(*output_type); + if (output_precision != parsed_precision || output_scale != parsed_scale) + throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Function {} has wrong output type", getName()); #endif - func_node = checkDecimalOverflow(actions_dag, func_node, parsed_precision, parsed_scale); + + return func_node; } return convertNodeTypeIfNeeded(substrait_func, func_node, actions_dag); } @@ -313,10 +186,6 @@ class FunctionParserPlus final : public FunctionParserBinaryArithmetic String getName() const override { return name; } protected: - DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override - { - return DecimalType::resultAddSubstractDecimalType(p1, s1, p2, s2); - } DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override { return DecimalType::evalAddSubstractDecimalType(p1, s1, p2, s2); @@ -332,10 +201,6 @@ class FunctionParserMinus final : public FunctionParserBinaryArithmetic String getName() const override { return name; } protected: - DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override - { - return DecimalType::resultAddSubstractDecimalType(p1, s1, p2, s2); - } DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override { return DecimalType::evalAddSubstractDecimalType(p1, s1, p2, s2); @@ -350,10 +215,6 @@ class FunctionParserMultiply final : public FunctionParserBinaryArithmetic String getName() const override { return name; } protected: - DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override - { - return DecimalType::resultMultiplyDecimalType(p1, s1, p2, s2); - } DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override { return DecimalType::evalMultiplyDecimalType(p1, s1, p2, s2); @@ -368,10 +229,6 @@ class FunctionParserDivide final : public FunctionParserBinaryArithmetic String getName() const override { return name; } protected: - DecimalType internalResultType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override - { - return DecimalType::resultDivideDecimalType(p1, s1, p2, s2); - } DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override { return DecimalType::evalDividetDecimalType(p1, s1, p2, s2); @@ -386,8 +243,8 @@ class FunctionParserDivide final : public FunctionParserBinaryArithmetic if (isDecimal(removeNullable(left_arg->result_type)) || isDecimal(removeNullable(right_arg->result_type))) return toFunctionNode(actions_dag, "sparkDivideDecimal", {left_arg, right_arg}); - else - return toFunctionNode(actions_dag, "sparkDivide", {left_arg, right_arg}); + + return toFunctionNode(actions_dag, "sparkDivide", {left_arg, right_arg}); } }; diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala index ff63a1726393..148cc4e609ce 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala @@ -273,9 +273,7 @@ object DecimalArithmeticUtil { } def checkAllowDecimalArithmetic(): Unit = { - // PrecisionLoss=true: velox support / ch not support - // PrecisionLoss=false: velox not support / ch support - // TODO ch support PrecisionLoss=true + // PrecisionLoss=false: velox not support if (!BackendsApiManager.getSettings.allowDecimalArithmetic) { throw new GlutenNotSupportException( s"Not support ${SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key} " + diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala index 494b67573a1a..ee765ed36099 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala @@ -116,7 +116,6 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait { .config(GlutenConfig.GLUTEN_LIB_PATH, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .config(GlutenConfig.UT_STATISTIC.key, "true") - .config("spark.sql.decimalOperations.allowPrecisionLoss", "false") .getOrCreate() } else { sparkBuilder From e5f1e5edfb43cd317e9f890ef011b6bc1d9cabf3 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 8 May 2024 23:41:10 +0800 Subject: [PATCH 031/402] [VL] Daily Update Velox Version (2024_05_08) (#5647) Velox changes ``` ac553396c by Ke, Fix PartitionIdGenerator's multiplier initialization for VectorHasher (9713) 7cfb42fb1 by joey.ljy, Fix the computation of hash tag in hash table doc (9699) ccbb72e50 by xiaoxmeng, Fix the lock order issue caused by value operator init (9738) 779500052 by PHILO-HE, Fix stemmer library when cmake < 3.18 is used (9670) b63a8691a by Zuyu ZHANG, Update README for build on MacOS (9717) ad25e87bb by xiaoxmeng, Avoid unnecessary reclaim on non-reclaimable query (9737) 1fd2bc960 by Richard Barnes, Change `result_of` to `invoke_result` in velox/vector/fuzzer/GeneratorSpec.h (9720) 668d57842 by xiaoxmeng, SSD cache code cleanup (9724) 591112932 by xiaoxmeng, Parallelize local memory arbitration (9649) ec4d2ec1f by Pedro Eugenio Rocha Pedreira, Improve evaluateOnce() helper function (9708) d41f654b3 by Zac Wen, Revert "Prevent cache write from exceeding IOV_MAX (9438)" (9722) 0c4dad14a by Jimmy Lu, Clean up selective file reader framework (9704) ``` --- .github/workflows/velox_velox_ut.yml | 8 +++----- ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/velox_velox_ut.yml b/.github/workflows/velox_velox_ut.yml index f1a1f9371d6e..efdf512983d6 100644 --- a/.github/workflows/velox_velox_ut.yml +++ b/.github/workflows/velox_velox_ut.yml @@ -19,10 +19,9 @@ name: Velox backend Velox Unit test on: pull_request: paths: - # TODO: wait to fix - # - '.github/workflows/velox_velox_ut.yml' + - '.github/workflows/velox_velox_ut.yml' - 'dev/**' - # - 'ep/**' #get_velox change + - 'ep/**' #get_velox change concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -65,5 +64,4 @@ jobs: - name: Run Tests run: | - ccache -c - cd ${GITHUB_WORKSPACE}/ep/build-velox/build/velox_ep/_build/release && ctest -E velox_cache_test -j 4 --output-on-failure --no-tests=error + cd ${GITHUB_WORKSPACE}/ep/build-velox/build/velox_ep/_build/release && ctest -E "velox_cache_test|velox_exec_test" -j 4 --output-on-failure --no-tests=error diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 987934204ac5..5ef78bb32958 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_07 +VELOX_BRANCH=2024_05_08 VELOX_HOME="" #Set on run gluten on HDFS From a7de1955a507c185fd7c0ef44a3a36b713115582 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 9 May 2024 12:50:57 +0800 Subject: [PATCH 032/402] [GLUTEN-5656][CORE] Avoid executing subqueries with complex data type during validation Fixes #5656 --- .../expression/ScalarSubqueryTransformer.scala | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala index 61b1e4382e54..534bde3b3a48 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala @@ -20,17 +20,13 @@ import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.{BaseSubqueryExec, ScalarSubquery} -import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} case class ScalarSubqueryTransformer(plan: BaseSubqueryExec, exprId: ExprId, query: ScalarSubquery) extends ExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { // don't trigger collect when in validation phase - if ( - TransformerState.underValidationState && - !valueSensitiveDataType(query.dataType) - ) { + if (TransformerState.underValidationState) { return ExpressionBuilder.makeLiteral(null, query.dataType, true) } // the first column in first row from `query`. @@ -50,16 +46,4 @@ case class ScalarSubqueryTransformer(plan: BaseSubqueryExec, exprId: ExprId, que } ExpressionBuilder.makeLiteral(result, query.dataType, result == null) } - - /** - * DataTypes which supported or not depend on actual value - * - * @param dataType - * @return - */ - def valueSensitiveDataType(dataType: DataType): Boolean = { - dataType.isInstanceOf[MapType] || - dataType.isInstanceOf[ArrayType] || - dataType.isInstanceOf[StructType] - } } From 53858ec3ded3cee2e675a1f729bee37718b57dd1 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Thu, 9 May 2024 13:45:17 +0800 Subject: [PATCH 033/402] [GLUTEN-5630][VL] Decrease peak memory by taking freeBytes into account (#5635) --- cpp/core/jni/JniCommon.h | 18 +++++++---- cpp/core/memory/AllocationListener.h | 8 +++++ cpp/core/memory/HbwAllocator.cc | 4 +++ cpp/core/memory/HbwAllocator.h | 2 ++ cpp/core/memory/MemoryAllocator.cc | 13 ++++++++ cpp/core/memory/MemoryAllocator.h | 7 ++++ cpp/velox/memory/VeloxMemoryManager.cc | 44 +++++++++++++++++--------- 7 files changed, 75 insertions(+), 21 deletions(-) diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index bda5fc1dfcb9..29c38689c67e 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -339,6 +339,14 @@ class SparkAllocationListener final : public gluten::AllocationListener { updateReservation(size); } + int64_t currentBytes() override { + return bytesReserved_; + } + + int64_t peakBytes() override { + return maxBytesReserved_; + } + private: int64_t reserve(int64_t diff) { std::lock_guard lock(mutex_); @@ -352,9 +360,7 @@ class SparkAllocationListener final : public gluten::AllocationListener { } int64_t bytesGranted = (newBlockCount - blocksReserved_) * blockSize_; blocksReserved_ = newBlockCount; - if (bytesReserved_ > maxBytesReserved_) { - maxBytesReserved_ = bytesReserved_; - } + maxBytesReserved_ = std::max(maxBytesReserved_, bytesReserved_); return bytesGranted; } @@ -368,10 +374,10 @@ class SparkAllocationListener final : public gluten::AllocationListener { if (granted < 0) { env->CallLongMethod(jListenerGlobalRef_, jUnreserveMethod_, -granted); checkException(env); - return; + } else { + env->CallLongMethod(jListenerGlobalRef_, jReserveMethod_, granted); + checkException(env); } - env->CallLongMethod(jListenerGlobalRef_, jReserveMethod_, granted); - checkException(env); } JavaVM* vm_; diff --git a/cpp/core/memory/AllocationListener.h b/cpp/core/memory/AllocationListener.h index 04290c8c3b52..23015e1a04ee 100644 --- a/cpp/core/memory/AllocationListener.h +++ b/cpp/core/memory/AllocationListener.h @@ -32,6 +32,14 @@ class AllocationListener { // Value of diff can be either positive or negative virtual void allocationChanged(int64_t diff) = 0; + virtual int64_t currentBytes() { + return 0; + } + + virtual int64_t peakBytes() { + return 0; + } + protected: AllocationListener() = default; }; diff --git a/cpp/core/memory/HbwAllocator.cc b/cpp/core/memory/HbwAllocator.cc index 9f2c5d6b6208..ef0dc82b8f93 100644 --- a/cpp/core/memory/HbwAllocator.cc +++ b/cpp/core/memory/HbwAllocator.cc @@ -85,4 +85,8 @@ int64_t HbwMemoryAllocator::getBytes() const { return bytes_; } +int64_t HbwMemoryAllocator::peakBytes() const { + return 0; +} + } // namespace gluten diff --git a/cpp/core/memory/HbwAllocator.h b/cpp/core/memory/HbwAllocator.h index 461f256828be..e50a71bd56b7 100644 --- a/cpp/core/memory/HbwAllocator.h +++ b/cpp/core/memory/HbwAllocator.h @@ -39,6 +39,8 @@ class HbwMemoryAllocator final : public MemoryAllocator { int64_t getBytes() const override; + int64_t peakBytes() const override; + private: std::atomic_int64_t bytes_{0}; }; diff --git a/cpp/core/memory/MemoryAllocator.cc b/cpp/core/memory/MemoryAllocator.cc index 218534cd86c0..6bcb9926eb45 100644 --- a/cpp/core/memory/MemoryAllocator.cc +++ b/cpp/core/memory/MemoryAllocator.cc @@ -29,6 +29,7 @@ bool ListenableMemoryAllocator::allocate(int64_t size, void** out) { } if (succeed) { bytes_ += size; + peakBytes_ = std::max(peakBytes_, bytes_.load()); } return succeed; } @@ -41,6 +42,7 @@ bool ListenableMemoryAllocator::allocateZeroFilled(int64_t nmemb, int64_t size, } if (succeed) { bytes_ += size * nmemb; + peakBytes_ = std::max(peakBytes_, bytes_.load()); } return succeed; } @@ -53,6 +55,7 @@ bool ListenableMemoryAllocator::allocateAligned(uint64_t alignment, int64_t size } if (succeed) { bytes_ += size; + peakBytes_ = std::max(peakBytes_, bytes_.load()); } return succeed; } @@ -66,6 +69,7 @@ bool ListenableMemoryAllocator::reallocate(void* p, int64_t size, int64_t newSiz } if (succeed) { bytes_ += diff; + peakBytes_ = std::max(peakBytes_, bytes_.load()); } return succeed; } @@ -84,6 +88,7 @@ bool ListenableMemoryAllocator::reallocateAligned( } if (succeed) { bytes_ += diff; + peakBytes_ = std::max(peakBytes_, bytes_.load()); } return succeed; } @@ -104,6 +109,10 @@ int64_t ListenableMemoryAllocator::getBytes() const { return bytes_; } +int64_t ListenableMemoryAllocator::peakBytes() const { + return peakBytes_; +} + bool StdMemoryAllocator::allocate(int64_t size, void** out) { *out = std::malloc(size); bytes_ += size; @@ -160,6 +169,10 @@ int64_t StdMemoryAllocator::getBytes() const { return bytes_; } +int64_t StdMemoryAllocator::peakBytes() const { + return 0; +} + std::shared_ptr defaultMemoryAllocator() { #if defined(GLUTEN_ENABLE_HBM) static std::shared_ptr alloc = HbwMemoryAllocator::newInstance(); diff --git a/cpp/core/memory/MemoryAllocator.h b/cpp/core/memory/MemoryAllocator.h index f9d5948fb223..a322c9190f9b 100644 --- a/cpp/core/memory/MemoryAllocator.h +++ b/cpp/core/memory/MemoryAllocator.h @@ -41,6 +41,8 @@ class MemoryAllocator { virtual bool free(void* p, int64_t size) = 0; virtual int64_t getBytes() const = 0; + + virtual int64_t peakBytes() const = 0; }; class ListenableMemoryAllocator final : public MemoryAllocator { @@ -63,10 +65,13 @@ class ListenableMemoryAllocator final : public MemoryAllocator { int64_t getBytes() const override; + int64_t peakBytes() const override; + private: MemoryAllocator* delegated_; AllocationListener* listener_; std::atomic_int64_t bytes_{0}; + int64_t peakBytes_{0}; }; class StdMemoryAllocator final : public MemoryAllocator { @@ -85,6 +90,8 @@ class StdMemoryAllocator final : public MemoryAllocator { int64_t getBytes() const override; + int64_t peakBytes() const override; + private: std::atomic_int64_t bytes_{0}; }; diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 93eb93f6bed9..49edba4c4299 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -40,11 +40,8 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { } uint64_t growCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { - if (targetBytes == 0) { - return 0; - } - std::lock_guard l(mutex_); - return growPoolLocked(pool, targetBytes); + VELOX_CHECK_EQ(targetBytes, 0, "Gluten has set MemoryManagerOptions.memoryPoolInitCapacity to 0") + return 0; } uint64_t shrinkCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { @@ -56,12 +53,11 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { velox::memory::MemoryPool* pool, const std::vector>& candidatePools, uint64_t targetBytes) override { - GLUTEN_CHECK(candidatePools.size() == 1, "ListenableArbitrator should only be used within a single root pool"); + VELOX_CHECK_EQ(candidatePools.size(), 1, "ListenableArbitrator should only be used within a single root pool") auto candidate = candidatePools.back(); - GLUTEN_CHECK(pool->root() == candidate.get(), "Illegal state in ListenableArbitrator"); - { + VELOX_CHECK(pool->root() == candidate.get(), "Illegal state in ListenableArbitrator") { std::lock_guard l(mutex_); - growPoolLocked(pool, targetBytes); + growPoolLocked(pool->root(), targetBytes); } return true; } @@ -72,7 +68,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { bool allowSpill, bool allowAbort) override { facebook::velox::exec::MemoryReclaimer::Stats status; - GLUTEN_CHECK(pools.size() == 1, "Should shrink a single pool at a time"); + VELOX_CHECK_EQ(pools.size(), 1, "Gluten only has one root pool"); std::lock_guard l(mutex_); // FIXME: Do we have recursive locking for this mutex? auto pool = pools.at(0); const uint64_t oldCapacity = pool->capacity(); @@ -107,8 +103,12 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { " bytes although there is enough space, free bytes: " + std::to_string(freeBytes)); return 0; } - listener_->allocationChanged(bytes); - return pool->grow(bytes, bytes); + auto reclaimedFreeBytes = pool->shrink(0); + auto neededBytes = bytes - reclaimedFreeBytes; + listener_->allocationChanged(neededBytes); + auto ret = pool->grow(bytes, bytes); + VELOX_CHECK(ret, "{} failed to grow {} bytes", pool->name(), velox::succinctBytes(bytes)) + return ret; } uint64_t releaseMemoryLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { @@ -179,18 +179,25 @@ VeloxMemoryManager::VeloxMemoryManager( } namespace { -MemoryUsageStats collectMemoryUsageStatsInternal(const velox::memory::MemoryPool* pool) { +MemoryUsageStats collectVeloxMemoryUsageStats(const velox::memory::MemoryPool* pool) { MemoryUsageStats stats; stats.set_current(pool->currentBytes()); stats.set_peak(pool->peakBytes()); // walk down root and all children pool->visitChildren([&](velox::memory::MemoryPool* pool) -> bool { - stats.mutable_children()->emplace(pool->name(), collectMemoryUsageStatsInternal(pool)); + stats.mutable_children()->emplace(pool->name(), collectVeloxMemoryUsageStats(pool)); return true; }); return stats; } +MemoryUsageStats collectGlutenAllocatorMemoryUsageStats(const MemoryAllocator* allocator) { + MemoryUsageStats stats; + stats.set_current(allocator->getBytes()); + stats.set_peak(allocator->peakBytes()); + return stats; +} + int64_t shrinkVeloxMemoryPool(velox::memory::MemoryManager* mm, velox::memory::MemoryPool* pool, int64_t size) { std::string poolName{pool->root()->name() + "/" + pool->name()}; std::string logPrefix{"Shrink[" + poolName + "]: "}; @@ -208,7 +215,14 @@ int64_t shrinkVeloxMemoryPool(velox::memory::MemoryManager* mm, velox::memory::M } // namespace const MemoryUsageStats VeloxMemoryManager::collectMemoryUsageStats() const { - return collectMemoryUsageStatsInternal(veloxAggregatePool_.get()); + MemoryUsageStats stats; + stats.set_current(listener_->currentBytes()); + stats.set_peak(listener_->peakBytes()); + stats.mutable_children()->emplace( + "gluten::MemoryAllocator", collectGlutenAllocatorMemoryUsageStats(glutenAlloc_.get())); + stats.mutable_children()->emplace( + veloxAggregatePool_->name(), collectVeloxMemoryUsageStats(veloxAggregatePool_.get())); + return stats; } const int64_t VeloxMemoryManager::shrink(int64_t size) { From a9fa7722405ddf72cc1aaba026aafebaf771748e Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 9 May 2024 14:02:36 +0800 Subject: [PATCH 034/402] [VL] Substrait-to-Velox: Support nested complex type signature parsing --- .../substrait/VeloxSubstraitSignature.cc | 103 ++++++++++++------ .../tests/VeloxSubstraitSignatureTest.cc | 7 ++ 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/cpp/velox/substrait/VeloxSubstraitSignature.cc b/cpp/velox/substrait/VeloxSubstraitSignature.cc index 34e0df6de2fd..ee7c5f513414 100644 --- a/cpp/velox/substrait/VeloxSubstraitSignature.cc +++ b/cpp/velox/substrait/VeloxSubstraitSignature.cc @@ -72,6 +72,48 @@ std::string VeloxSubstraitSignature::toSubstraitSignature(const TypePtr& type) { } } +namespace { +using index = std::string::size_type; + +index findEnclosingPos(std::string text, index from, char left, char right) { + VELOX_CHECK(left != right) + VELOX_CHECK(text.at(from) == left) + int32_t stackedLeftChars = 0; + for (index idx = from; idx < text.size(); idx++) { + const char ch = text.at(idx); + if (ch == left) { + stackedLeftChars++; + } + if (ch == right) { + stackedLeftChars--; + } + if (stackedLeftChars == 0) { + return idx; + } + } + VELOX_FAIL("Unable to find enclose character from text: " + text) +} + +index findSansNesting(std::string text, index from, char target, char left, char right) { + VELOX_CHECK(left != right) + VELOX_CHECK(target != left && target != right) + int32_t stackedLeftChars = 0; + for (index idx = from; idx < text.size(); idx++) { + const char ch = text.at(idx); + if (ch == left) { + stackedLeftChars++; + } + if (ch == right) { + stackedLeftChars--; + } + if (ch == target && stackedLeftChars == 0) { + return idx; + } + } + return std::string::npos; +} +} // namespace + TypePtr VeloxSubstraitSignature::fromSubstraitSignature(const std::string& signature) { if (signature == "bool") { return BOOLEAN(); @@ -123,7 +165,7 @@ TypePtr VeloxSubstraitSignature::fromSubstraitSignature(const std::string& signa auto parseNestedTypeSignature = [&](const std::string& signature) -> std::vector { auto start = signature.find_first_of('<'); - auto end = signature.find_last_of('>'); + auto end = findEnclosingPos(signature, start, '<', '>'); VELOX_CHECK( end - start > 1, "Native validation failed due to: more information is needed to create nested type for {}", @@ -132,30 +174,25 @@ TypePtr VeloxSubstraitSignature::fromSubstraitSignature(const std::string& signa std::string childrenTypes = signature.substr(start + 1, end - start - 1); // Split the types with delimiter. - std::string delimiter = ","; - std::size_t pos; + const char delimiter = ','; std::vector types; - while ((pos = childrenTypes.find(delimiter)) != std::string::npos) { - auto typeStr = childrenTypes.substr(0, pos); - std::size_t endPos = pos; - if (startWith(typeStr, "dec") || startWith(typeStr, "struct") || startWith(typeStr, "map") || - startWith(typeStr, "list")) { - endPos = childrenTypes.find(">") + 1; - if (endPos > pos) { - typeStr += childrenTypes.substr(pos, endPos - pos); - } else { - // For nested case, the end '>' could missing, - // so the last position is treated as end. - typeStr += childrenTypes.substr(pos); - endPos = childrenTypes.size(); - } + size_t typeStart = 0; + while (true) { + if (typeStart == childrenTypes.size()) { + break; + } + VELOX_CHECK(typeStart < childrenTypes.size()) + const size_t typeEnd = findSansNesting(childrenTypes, typeStart, delimiter, '<', '>'); + if (typeEnd == std::string::npos) { + std::string typeStr = childrenTypes.substr(typeStart); + types.emplace_back(fromSubstraitSignature(typeStr)); + break; } + std::string typeStr = childrenTypes.substr(typeStart, typeEnd - typeStart); types.emplace_back(fromSubstraitSignature(typeStr)); - childrenTypes.erase(0, endPos + delimiter.length()); - } - if (childrenTypes.size() > 0 && !startWith(childrenTypes, ">")) { - types.emplace_back(fromSubstraitSignature(childrenTypes)); + typeStart = typeEnd + 1; } + return types; }; @@ -172,6 +209,10 @@ TypePtr VeloxSubstraitSignature::fromSubstraitSignature(const std::string& signa if (startWith(signature, "struct")) { // Struct type name is in the format of struct. auto types = parseNestedTypeSignature(signature); + if (types.empty()) { + VELOX_UNSUPPORTED( + "VeloxSubstraitSignature::fromSubstraitSignature: Unrecognizable struct type signature {}.", signature); + } std::vector names(types.size()); for (int i = 0; i < types.size(); i++) { names[i] = ""; @@ -183,22 +224,20 @@ TypePtr VeloxSubstraitSignature::fromSubstraitSignature(const std::string& signa // Map type name is in the format of map. auto types = parseNestedTypeSignature(signature); if (types.size() != 2) { - VELOX_UNSUPPORTED("Substrait type signature conversion to Velox type not supported for {}.", signature); + VELOX_UNSUPPORTED( + "VeloxSubstraitSignature::fromSubstraitSignature: Unrecognizable map type signature {}.", signature); } return MAP(std::move(types)[0], std::move(types)[1]); } if (startWith(signature, "list")) { - auto listStart = signature.find_first_of('<'); - auto listEnd = signature.find_last_of('>'); - VELOX_CHECK( - listEnd - listStart > 1, - "Native validation failed due to: more information is needed to create ListType: {}", - signature); - - auto elementTypeStr = signature.substr(listStart + 1, listEnd - listStart - 1); - auto elementType = fromSubstraitSignature(elementTypeStr); - return ARRAY(elementType); + // Array type name is in the format of list. + auto types = parseNestedTypeSignature(signature); + if (types.size() != 1) { + VELOX_UNSUPPORTED( + "VeloxSubstraitSignature::fromSubstraitSignature: Unrecognizable list type signature {}.", signature); + } + return ARRAY(std::move(types)[0]); } VELOX_UNSUPPORTED("Substrait type signature conversion to Velox type not supported for {}.", signature); diff --git a/cpp/velox/tests/VeloxSubstraitSignatureTest.cc b/cpp/velox/tests/VeloxSubstraitSignatureTest.cc index d6db661f76cd..cb62f9764913 100644 --- a/cpp/velox/tests/VeloxSubstraitSignatureTest.cc +++ b/cpp/velox/tests/VeloxSubstraitSignatureTest.cc @@ -138,6 +138,13 @@ TEST_F(VeloxSubstraitSignatureTest, fromSubstraitSignature) { ASSERT_EQ(type->childAt(0)->childAt(0)->childAt(1)->kind(), TypeKind::VARCHAR); type = fromSubstraitSignature("struct>>>"); ASSERT_EQ(type->childAt(0)->childAt(0)->childAt(1)->kind(), TypeKind::HUGEINT); + type = fromSubstraitSignature("struct,list,map>>>"); + ASSERT_EQ(type->childAt(0)->kind(), TypeKind::REAL); + ASSERT_EQ(type->childAt(1)->childAt(0)->childAt(0)->kind(), TypeKind::TINYINT); + ASSERT_EQ(type->childAt(1)->childAt(0)->childAt(1)->kind(), TypeKind::HUGEINT); + ASSERT_EQ(type->childAt(1)->childAt(0)->childAt(2)->childAt(0)->kind(), TypeKind::INTEGER); + ASSERT_EQ(type->childAt(1)->childAt(0)->childAt(3)->childAt(0)->kind(), TypeKind::SMALLINT); + ASSERT_EQ(type->childAt(1)->childAt(0)->childAt(3)->childAt(1)->kind(), TypeKind::INTEGER); ASSERT_ANY_THROW(fromSubstraitSignature("other")->kind()); // Map type test. From 69fa591d48f6e085e2b4059c1588b845d34c5d1f Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Thu, 9 May 2024 11:33:44 +0530 Subject: [PATCH 035/402] [VL] Remove spark.gluten.sql.columnar.backend.lib (#5671) [VL] Remove spark.gluten.sql.columnar.backend.lib. --- docs/get-started/Work-with-pyspark.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/get-started/Work-with-pyspark.ipynb b/docs/get-started/Work-with-pyspark.ipynb index 40c98d333331..23d7c7742f56 100644 --- a/docs/get-started/Work-with-pyspark.ipynb +++ b/docs/get-started/Work-with-pyspark.ipynb @@ -41,7 +41,6 @@ "conf.set(\"spark.driver.extraClassPath\", nativesql_jars)\n", "conf.set(\"spark.executor.extraClassPath\", nativesql_jars)\n", "conf.set(\"spark.plugins\", \"org.apache.gluten.GlutenPlugin\")\n", - "conf.set(\"spark.gluten.sql.columnar.backend.lib\", \"velox\")\n", "conf.set(\"spark.gluten.loadLibFromJar\", \"false\")\n", "conf.set(\"spark.shuffle.manager\", \"org.apache.spark.shuffle.sort.ColumnarShuffleManager\")\n", "sc = SparkContext(conf=conf)\n", From 5da9f6ea1f8565c13b088fb104a723d01eff7912 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Thu, 9 May 2024 03:16:48 -0500 Subject: [PATCH 036/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240509) (#5666) Co-authored-by: kyligence-git We need merge since rebase failed with https://github.com/ClickHouse/ClickHouse/pull/63488 --- cpp-ch/clickhouse.version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 9f4ee9241ba7..a692c5666aef 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240508 -CH_COMMIT=1dfaf7ffeaa \ No newline at end of file +CH_BRANCH=rebase_ch/20240509 +CH_COMMIT=81ee5ff107b \ No newline at end of file From 949650969d32d9149f438b856c27c6c609cd1946 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Thu, 9 May 2024 20:40:15 +0800 Subject: [PATCH 037/402] [VL] Defer debug log generation (#5672) Memory dump message will be generated whether debug log enable or not, add check to defer it and save cpu cycles. --- .../memory/nmm/NativeMemoryManager.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java index 95e9843bea0b..230a7342e87e 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java @@ -95,20 +95,22 @@ private static native long create( @Override public void release() throws Exception { - LOGGER.debug( - SparkMemoryUtil.prettyPrintStats( - "About to release memory manager, usage dump:", - new KnownNameAndStats() { - @Override - public String name() { - return name; - } - - @Override - public MemoryUsageStats stats() { - return collectMemoryUsage(); - } - })); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + SparkMemoryUtil.prettyPrintStats( + "About to release memory manager, usage dump:", + new KnownNameAndStats() { + @Override + public String name() { + return name; + } + + @Override + public MemoryUsageStats stats() { + return collectMemoryUsage(); + } + })); + } release(nativeInstanceHandle); if (listener.getUsedBytes() != 0) { LOGGER.warn( From 24d949e617103a98e339ecbedc1c73c35a48a513 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 10 May 2024 04:20:00 +0800 Subject: [PATCH 038/402] [VL] Daily Update Velox Version (2024_05_09) (#5664) Upstream Velox's New Commits: f53687563 by zjuwangg, Add more test method in Filter::AlwaysFalse class (9718) 48cfbe3e7 by Zac Wen, Prevent duplicates in writable regions (9746) 87ea7d5fb by Daniel Munoz, Create howMuchToSkip method in UnitLoaderTools (9706) 16e2c1375 by Daniel Munoz, UnitLoader supports initial rowsToSkip (9703) dec2556b8 by Ubuntu, Add barrier at kernel end and scatterBits (9745) 20be1936b by PHILO-HE, Register some re-usable Presto functions for Spark (9425) 798a12736 by Jimmy Lu, Fix crash in ArrayVectorBase::copyRangesImpl in case target offsets or sizes is nullptr (9725) bc6632ac9 by Karteekmurthys, Add decimal support for min_by and max_by functions (8723) 5c4903fe2 by Zac Wen, Handle ssd cache region score overflow (9709) 49b124665 by Sergey Pershin, Improve instrumentation around Zombie Task detection. (9663) --- .github/workflows/velox_docker_cache.yml | 68 +++++++++++++++++++ .../gluten/execution/TestOperator.scala | 2 +- .../Substrait2VeloxPlanConversionTest.cc | 6 +- ep/build-velox/src/get_velox.sh | 2 +- 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 969053657b1a..d79b5e717154 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -51,3 +51,71 @@ jobs: with: path: ./cpp/build/releases/ key: cache-velox-build-${{ hashFiles('./cache-key') }} + + ccache-native-lib-ubuntu-velox-ut: + runs-on: ubuntu-20.04 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx + steps: + - uses: actions/checkout@v2 + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-ubuntu-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' + - name: Build Gluten velox third party + run: | + rm -rf /opt/miniconda-for-velox/ + cd ep/build-velox/src && \ + ./get_velox.sh + cd ../build/velox_ep/ + git reset --hard + make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" + + - name: CCache after + run: | + ccache -vs + + - uses: actions/cache/save@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-ubuntu-release-default + ccache-native-lib-centos-velox-ut: + runs-on: ubuntu-20.04 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + container: ghcr.io/facebookincubator/velox-dev:circleci-avx + steps: + - uses: actions/checkout@v2 + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' + - name: Build Gluten velox third party + run: | + rm -rf /opt/miniconda-for-velox/ + cd ep/build-velox/src && \ + ./get_velox.sh + cd ../build/velox_ep/ + source /opt/rh/gcc-toolset-9/enable + git reset --hard + make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" + + - name: CCache after + run: | + ccache -vs + + - uses: actions/cache/save@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index a14a5b7e78de..920b8e2bde9f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -630,7 +630,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } assert(wholeStageTransformers.size == 3) val nativePlanString = wholeStageTransformers.head.nativePlanString() - assert(nativePlanString.contains("Aggregation[SINGLE")) + assert(nativePlanString.contains("Aggregation[1][SINGLE")) assert(nativePlanString.contains("ValueStream")) assert(wholeStageTransformers(1).nativePlanString().contains("ValueStream")) assert(wholeStageTransformers.last.nativePlanString().contains("TableScan")) diff --git a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc index e64331610f9e..841514261859 100644 --- a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc +++ b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc @@ -258,8 +258,8 @@ TEST_F(Substrait2VeloxPlanConversionTest, ifthenTest) { // Convert to Velox PlanNode. auto planNode = planConverter_->toVeloxPlan(substraitPlan, std::vector<::substrait::ReadRel_LocalFiles>{split}); ASSERT_EQ( - "-- Project[expressions: ] -> \n " - "-- TableScan[table: hive_table, range filters: [(hd_demo_sk, Filter(IsNotNull, deterministic, null not allowed))," + "-- Project[1][expressions: ] -> \n " + "-- TableScan[0][table: hive_table, range filters: [(hd_demo_sk, Filter(IsNotNull, deterministic, null not allowed))," " (hd_vehicle_count, BigintRange: [1, 9223372036854775807] no nulls)], remaining filter: " "(and(or(equalto(\"hd_buy_potential\",\">10000\"),equalto(\"hd_buy_potential\",\"unknown\"))," "if(greaterthan(\"hd_vehicle_count\",0),greaterthan(divide(cast \"hd_dep_count\" as DOUBLE," @@ -279,7 +279,7 @@ TEST_F(Substrait2VeloxPlanConversionTest, filterUpper) { // Convert to Velox PlanNode. auto planNode = planConverter_->toVeloxPlan(substraitPlan, std::vector<::substrait::ReadRel_LocalFiles>{split}); ASSERT_EQ( - "-- Project[expressions: ] -> \n -- TableScan[table: hive_table, range filters: " + "-- Project[1][expressions: ] -> \n -- TableScan[0][table: hive_table, range filters: " "[(key, BigintRange: [-2147483648, 2] no nulls)]] -> n0_0:INTEGER\n", planNode->toString(true, true)); } diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 5ef78bb32958..a3013dd49087 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_08 +VELOX_BRANCH=2024_05_09 VELOX_HOME="" #Set on run gluten on HDFS From 6c1a7478c3ba5632291a96420b76acf49cff5a2a Mon Sep 17 00:00:00 2001 From: Wei-Ting Chen Date: Fri, 10 May 2024 11:02:10 +0800 Subject: [PATCH 039/402] [DOC]add Gluten logo (#5680) * add Gluten logo * change img method --- README.md | 2 ++ docs/image/gluten-logo.svg | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/image/gluten-logo.svg diff --git a/README.md b/README.md index a99757e106d8..3f99a7cc7bf3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![Gluten](docs/image/gluten-logo.svg) + # Apache Gluten (Incubating): A Middle Layer for Offloading JVM-based SQL Engines' Execution to Native Engines [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8452/badge)](https://www.bestpractices.dev/projects/8452) diff --git a/docs/image/gluten-logo.svg b/docs/image/gluten-logo.svg new file mode 100644 index 000000000000..aeee80b748ba --- /dev/null +++ b/docs/image/gluten-logo.svg @@ -0,0 +1,9 @@ + + + + + + + + + From f6e882335feb99a69a42837cfbcf7e65554347dd Mon Sep 17 00:00:00 2001 From: Tengfei Huang Date: Fri, 10 May 2024 11:24:02 +0800 Subject: [PATCH 040/402] [GLUTEN-5662][VL] Fix literal array conversion with nested empty array/map ahead of non-empty (#5663) --- .../gluten/execution/VeloxLiteralSuite.scala | 2 + cpp/velox/substrait/SubstraitToVeloxExpr.cc | 38 ++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) mode change 100644 => 100755 cpp/velox/substrait/SubstraitToVeloxExpr.cc diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala index 0ad26aa03e09..cf2e7257f528 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxLiteralSuite.scala @@ -74,7 +74,9 @@ class VeloxLiteralSuite extends VeloxWholeStageTransformerSuite { test("Array Literal") { validateOffloadResult("SELECT array()") validateOffloadResult("SELECT array(array())") + validateOffloadResult("SELECT array(array(), array(1, 2))") validateOffloadResult("SELECT array(map())") + validateOffloadResult("SELECT array(map(), map('red', 1))") validateOffloadResult("SELECT array('Spark', '5')") validateOffloadResult("SELECT array(5, 1, -1)") validateOffloadResult("SELECT array(5S, 1S, -1S)") diff --git a/cpp/velox/substrait/SubstraitToVeloxExpr.cc b/cpp/velox/substrait/SubstraitToVeloxExpr.cc old mode 100644 new mode 100755 index 03e91ac42493..fb4861e4fee7 --- a/cpp/velox/substrait/SubstraitToVeloxExpr.cc +++ b/cpp/velox/substrait/SubstraitToVeloxExpr.cc @@ -430,11 +430,21 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector( } case ::substrait::Expression_Literal::LiteralTypeCase::kIntervalDayToSecond: return constructFlatVector(elementAtFunc, childSize, INTERVAL_DAY_TIME(), pool_); + // Handle EmptyList and List together since the children could be either case. + case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyList: case ::substrait::Expression_Literal::LiteralTypeCase::kList: { ArrayVectorPtr elements; for (int i = 0; i < childSize; i++) { - auto element = elementAtFunc(i); - ArrayVectorPtr grandVector = literalsToArrayVector(element); + auto child = elementAtFunc(i); + auto childType = child.literal_type_case(); + ArrayVectorPtr grandVector; + + if (childType == ::substrait::Expression_Literal::LiteralTypeCase::kEmptyList) { + auto elementType = SubstraitParser::parseType(child.empty_list().type()); + grandVector = makeEmptyArrayVector(pool_, elementType); + } else { + grandVector = literalsToArrayVector(child); + } if (!elements) { elements = grandVector; } else { @@ -443,11 +453,22 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector( } return elements; } + // Handle EmptyMap and Map together since the children could be either case. + case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyMap: case ::substrait::Expression_Literal::LiteralTypeCase::kMap: { MapVectorPtr mapVector; for (int i = 0; i < childSize; i++) { - auto element = elementAtFunc(i); - MapVectorPtr grandVector = literalsToMapVector(element); + auto child = elementAtFunc(i); + auto childType = child.literal_type_case(); + MapVectorPtr grandVector; + + if (childType == ::substrait::Expression_Literal::LiteralTypeCase::kEmptyMap) { + auto keyType = SubstraitParser::parseType(child.empty_map().key()); + auto valueType = SubstraitParser::parseType(child.empty_map().value()); + grandVector = makeEmptyMapVector(pool_, keyType, valueType); + } else { + grandVector = literalsToMapVector(child); + } if (!mapVector) { mapVector = grandVector; } else { @@ -469,15 +490,6 @@ VectorPtr SubstraitVeloxExprConverter::literalsToVector( } return rowVector; } - case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyList: { - auto elementType = SubstraitParser::parseType(childLiteral.empty_list().type()); - return BaseVector::wrapInConstant(1, 0, makeEmptyArrayVector(pool_, elementType)); - } - case ::substrait::Expression_Literal::LiteralTypeCase::kEmptyMap: { - auto keyType = SubstraitParser::parseType(childLiteral.empty_map().key()); - auto valueType = SubstraitParser::parseType(childLiteral.empty_map().value()); - return BaseVector::wrapInConstant(1, 0, makeEmptyMapVector(pool_, keyType, valueType)); - } default: auto veloxType = getScalarType(elementAtFunc(0)); if (veloxType) { From a7e7f657866c39e163290413c2bceb3d1a04f019 Mon Sep 17 00:00:00 2001 From: Kerwin Zhang Date: Fri, 10 May 2024 12:07:33 +0800 Subject: [PATCH 041/402] [VL] Use the default pip3.6 from the alinux3 in the build of velox (#5676) --- ep/build-velox/src/get_velox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index a3013dd49087..859bb2cbc1f3 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -205,6 +205,7 @@ function process_setup_alinux3 { sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|' scripts/setup-centos8.sh sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|' scripts/setup-centos8.sh sed -i 's/python39 python39-devel python39-pip //g' scripts/setup-centos8.sh + sed -i 's/pip3.9/pip3.6/g' scripts/setup-centos8.sh sed -i "s/\${CMAKE_INSTALL_LIBDIR}/lib64/" third_party/CMakeLists.txt } From 6fba94991a4232839654edfbcb1668a5df45203c Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Fri, 10 May 2024 12:47:10 +0800 Subject: [PATCH 042/402] [GLUTEN-5673][VL] Fix arbitrator grow logic when exist concurrent memory request (#5674) --- cpp/velox/memory/VeloxMemoryManager.cc | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 49edba4c4299..1347bb950bdf 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -46,7 +46,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { uint64_t shrinkCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { std::lock_guard l(mutex_); - return releaseMemoryLocked(pool, targetBytes); + return shrinkCapacityLocked(pool, targetBytes); } bool growCapacity( @@ -55,10 +55,10 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { uint64_t targetBytes) override { VELOX_CHECK_EQ(candidatePools.size(), 1, "ListenableArbitrator should only be used within a single root pool") auto candidate = candidatePools.back(); - VELOX_CHECK(pool->root() == candidate.get(), "Illegal state in ListenableArbitrator") { - std::lock_guard l(mutex_); - growPoolLocked(pool->root(), targetBytes); - } + VELOX_CHECK(pool->root() == candidate.get(), "Illegal state in ListenableArbitrator"); + + std::lock_guard l(mutex_); + growCapacityLocked(pool->root(), targetBytes); return true; } @@ -90,28 +90,29 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { } private: - uint64_t growPoolLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { + void growCapacityLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { // Since // https://github.com/facebookincubator/velox/pull/9557/files#diff-436e44b7374032f8f5d7eb45869602add6f955162daa2798d01cc82f8725724dL812-L820, // We should pass bytes as parameter "reservationBytes" when calling ::grow. - const uint64_t freeBytes = pool->freeBytes(); - if (freeBytes >= bytes) { - bool reserved = pool->grow(0, bytes); - GLUTEN_CHECK( - reserved, - "Unexpected: Failed to reserve " + std::to_string(bytes) + - " bytes although there is enough space, free bytes: " + std::to_string(freeBytes)); - return 0; + auto freeByes = pool->freeBytes(); + if (freeByes > bytes) { + if (pool->grow(0, bytes)) { + return; + } } auto reclaimedFreeBytes = pool->shrink(0); auto neededBytes = bytes - reclaimedFreeBytes; listener_->allocationChanged(neededBytes); auto ret = pool->grow(bytes, bytes); - VELOX_CHECK(ret, "{} failed to grow {} bytes", pool->name(), velox::succinctBytes(bytes)) - return ret; + VELOX_CHECK( + ret, + "{} failed to grow {} bytes, current state {}", + pool->name(), + velox::succinctBytes(bytes), + pool->toString()) } - uint64_t releaseMemoryLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { + uint64_t shrinkCapacityLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { uint64_t freeBytes = pool->shrink(bytes); listener_->allocationChanged(-freeBytes); return freeBytes; From 04df29d76d89b309af8519a1fb79a4cdfe3bb7f6 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 10 May 2024 15:39:12 +0800 Subject: [PATCH 043/402] [VL] Daily Update Velox Version (2024_05_10) (#5678) Velox changes ``` 7d76b1d9e by Jialiang Tan, Propagate additional dwrf writer options (9752) 49c3ebb20 by gaoyangxiaozhu, Row index metadata column support for table scan (9174) 3e98d4080 by Jimmy Lu, Remove lint warnings in ParquetReaderTest (9760) dceaff0f2 by Jimmy Lu, Move E2EFilterTestBase into tests/utils to be reused by Nimble (9749) eb356fddc by chliang, Create row type for list children when there are >1 child fields (9533) c325d4fa6 by xiaoxmeng, Add async memory reclaim task to avoid recursive arbitration (9734) b8a26ce8e by Jialiang Tan, Add memory and cache stats to PeriodicStatsReporter (9723) 9ad161abf by Sergey Pershin, Keep weak pointers of Drivers closed by Task. (9751) ``` --- .github/workflows/velox_docker_cache.yml | 8 ++++++-- cpp/velox/memory/VeloxMemoryManager.cc | 2 ++ ep/build-velox/src/get_velox.sh | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index d79b5e717154..91676dbb6ade 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -74,7 +74,6 @@ jobs: cd ep/build-velox/src && \ ./get_velox.sh cd ../build/velox_ep/ - git reset --hard make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" - name: CCache after @@ -92,6 +91,12 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 + - name: Setup java and maven + run: | + yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz + mv apache-maven-3.8.8 /usr/lib/maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -108,7 +113,6 @@ jobs: ./get_velox.sh cd ../build/velox_ep/ source /opt/rh/gcc-toolset-9/enable - git reset --hard make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" - name: CCache after diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 1347bb950bdf..f49beaccd264 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -53,6 +53,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { velox::memory::MemoryPool* pool, const std::vector>& candidatePools, uint64_t targetBytes) override { + velox::memory::ScopedMemoryArbitrationContext ctx(pool); VELOX_CHECK_EQ(candidatePools.size(), 1, "ListenableArbitrator should only be used within a single root pool") auto candidate = candidatePools.back(); VELOX_CHECK(pool->root() == candidate.get(), "Illegal state in ListenableArbitrator"); @@ -67,6 +68,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { uint64_t targetBytes, bool allowSpill, bool allowAbort) override { + velox::memory::ScopedMemoryArbitrationContext ctx(nullptr); facebook::velox::exec::MemoryReclaimer::Stats status; VELOX_CHECK_EQ(pools.size(), 1, "Gluten only has one root pool"); std::lock_guard l(mutex_); // FIXME: Do we have recursive locking for this mutex? diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 859bb2cbc1f3..46dc793ed423 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_09 +VELOX_BRANCH=2024_05_10 VELOX_HOME="" #Set on run gluten on HDFS From 5d8ac72966b9c23870654fe56ffe276579c93c15 Mon Sep 17 00:00:00 2001 From: Kerwin Zhang Date: Fri, 10 May 2024 16:28:42 +0800 Subject: [PATCH 044/402] [VL] Add -Wno-stringop-overflow for alinux3 (#5686) --- ep/build-velox/src/get_velox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 46dc793ed423..b54da5c68943 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -206,6 +206,7 @@ function process_setup_alinux3 { sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|' scripts/setup-centos8.sh sed -i 's/python39 python39-devel python39-pip //g' scripts/setup-centos8.sh sed -i 's/pip3.9/pip3.6/g' scripts/setup-centos8.sh + sed -i 's/ADDITIONAL_FLAGS=""/ADDITIONAL_FLAGS="-Wno-stringop-overflow"/g' scripts/setup-helper-functions.sh sed -i "s/\${CMAKE_INSTALL_LIBDIR}/lib64/" third_party/CMakeLists.txt } From c48e42685557dcce5b34d5cb26bdb210a3d9dbbc Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 10 May 2024 20:41:29 +0800 Subject: [PATCH 045/402] [GLUTEN-2620][VL] Enable compile_arrow_java by default to avoid invalid pointer error (#5648) Though the reported issue is on centos-7 by now, it's still possible that user encounters such issue on other OS. So let's directly enable this option by default. --- .github/workflows/velox_docker.yml | 105 +- dev/builddeps-veloxbe.sh | 2 +- .../ports/aws-sdk-cpp/fix-aws-root.patch | 22 + .../fix-awsmigrationhub-build.patch | 10 + dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch | 12 + .../ports/aws-sdk-cpp/generateFeatures.ps1 | 73 + .../lock-curl-http-and-tls-settings.patch | 20 + .../aws-sdk-cpp/patch-relocatable-rpath.patch | 12 + dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake | 101 ++ dev/vcpkg/ports/aws-sdk-cpp/usage | 12 + dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json | 38 + dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json | 1187 +++++++++++++++++ dev/vcpkg/vcpkg.json | 1 - docs/get-started/build-guide.md | 4 +- docs/velox-backend-troubleshooting.md | 34 +- ep/build-velox/src/modify_arrow.patch | 31 +- 16 files changed, 1591 insertions(+), 73 deletions(-) create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/fix-aws-root.patch create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/generateFeatures.ps1 create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/patch-relocatable-rpath.patch create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/usage create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 284bf9198862..194cea28727e 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -162,7 +162,6 @@ jobs: mv apache-maven-3.8.8 /usr/lib/maven - name: Set environment variables run: | - echo "MAVEN_HOME=/usr/lib/maven" >> $GITHUB_ENV echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV if [ "${{ matrix.java }}" = "java-17" ]; then echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV @@ -500,16 +499,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -545,8 +543,6 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ mvn -ntp test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files @@ -566,16 +562,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -595,9 +590,7 @@ jobs: mv sql shims/spark32/spark_home/ - name: Build and run unit test for Spark 3.2.2 (slow tests) run: | - cd $GITHUB_WORKSPACE/ && \ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin + cd $GITHUB_WORKSPACE/ mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark33: @@ -605,16 +598,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -644,10 +636,8 @@ jobs: pip3 install pandas pyarrow - name: Build and Run unit test for Spark 3.3.1 (other tests) run: | - cd $GITHUB_WORKSPACE/ && \ - export SPARK_SCALA_VERSION=2.12 && \ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.12 mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ mvn -ntp test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files @@ -663,16 +653,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget numactl-devel -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -692,9 +681,7 @@ jobs: mv sql shims/spark33/spark_home/ - name: Build and Run unit test for Spark 3.3.1 (slow tests) run: | - cd $GITHUB_WORKSPACE/ && \ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin + cd $GITHUB_WORKSPACE/ mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark34: @@ -702,16 +689,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -741,10 +727,8 @@ jobs: pip3 install pandas pyarrow - name: Build and Run unit test for Spark 3.4.2 (other tests) run: | - cd $GITHUB_WORKSPACE/ && \ - export SPARK_SCALA_VERSION=2.12 && \ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.12 mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ mvn -ntp test -Pspark-3.4 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files @@ -760,16 +744,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -790,8 +773,6 @@ jobs: - name: Build and Run unit test for Spark 3.4.2 (slow tests) run: | cd $GITHUB_WORKSPACE/ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark35: @@ -799,16 +780,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven && \ - export PATH=${PATH}:${MAVEN_HOME}/bin && \ cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -838,10 +818,8 @@ jobs: pip3 install pandas pyarrow - name: Build and Run unit test for Spark 3.5.1 (other tests) run: | - cd $GITHUB_WORKSPACE/ && \ - export SPARK_SCALA_VERSION=2.12 && \ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.12 mvn -ntp clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ mvn -ntp test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files @@ -856,16 +834,15 @@ jobs: container: ghcr.io/facebookincubator/velox-dev:circleci-avx steps: - uses: actions/checkout@v2 - - name: Setup java and maven + - name: Setup build dependency run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ + yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten velox third party run: | - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin cd ep/build-velox/src && \ ./get_velox.sh && \ source /opt/rh/gcc-toolset-9/enable && \ @@ -892,6 +869,4 @@ jobs: - name: Build and Run unit test for Spark 3.5.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - export MAVEN_HOME=/usr/lib/maven - export PATH=${PATH}:${MAVEN_HOME}/bin mvn -ntp clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ No newline at end of file diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index e90106ee77f4..232c36d4053c 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -31,7 +31,7 @@ VELOX_REPO="" VELOX_BRANCH="" VELOX_HOME="" VELOX_PARAMETER="" -COMPILE_ARROW_JAVA=OFF +COMPILE_ARROW_JAVA=ON # set default number of threads as cpu cores minus 2 if [[ "$(uname)" == "Darwin" ]]; then diff --git a/dev/vcpkg/ports/aws-sdk-cpp/fix-aws-root.patch b/dev/vcpkg/ports/aws-sdk-cpp/fix-aws-root.patch new file mode 100644 index 000000000000..da4365ad7218 --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/fix-aws-root.patch @@ -0,0 +1,22 @@ +diff --git a/cmake/AWSSDKConfig.cmake b/cmake/AWSSDKConfig.cmake +--- a/cmake/AWSSDKConfig.cmake (revision 2f90f9fd6c56460bd382243aa215fcddcb5883c8) ++++ b/cmake/AWSSDKConfig.cmake (date 1636913220527) +@@ -54,18 +54,14 @@ + string(REPLACE ";" "${AWS_MODULE_DIR};" SYSTEM_MODULE_PATH "${CMAKE_SYSTEM_PREFIX_PATH}${AWS_MODULE_DIR}") + list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH} ${SYSTEM_MODULE_PATH}) + +-# On Windows, dlls are treated as runtime target and installed in bindir + if (WIN32 AND AWSSDK_INSTALL_AS_SHARED_LIBS) +- set(AWSSDK_INSTALL_LIBDIR "${AWSSDK_INSTALL_BINDIR}") + # If installed CMake scripts are associated with dll library, define USE_IMPORT_EXPORT for customers + add_definitions(-DUSE_IMPORT_EXPORT) + endif() + + + # Compute the default installation root relative to this file. +-# from prefix/lib/cmake/AWSSDK/xx.cmake to prefix + get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +-get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) + get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) + get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) + get_filename_component(AWS_NATIVE_SDK_ROOT "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) diff --git a/dev/vcpkg/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch b/dev/vcpkg/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch new file mode 100644 index 000000000000..c2d98030136e --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch @@ -0,0 +1,10 @@ +diff --git a/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt b/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt +index a8a888d..574b985 100644 +--- a/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt ++++ b/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt +@@ -1,4 +1,4 @@ +-add_project(aws-cpp-sdk-AWSMigrationHub "C++ SDK for the AWS AWSMigrationHub service" aws-cpp-sdk-core) ++add_project(aws-cpp-sdk-awsmigrationhub "C++ SDK for the AWS AWSMigrationHub service" aws-cpp-sdk-core) + + file(GLOB AWS_AWSMIGRATIONHUB_HEADERS + "include/aws/AWSMigrationHub/*.h" diff --git a/dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch b/dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch new file mode 100644 index 000000000000..be4511ada4ef --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch @@ -0,0 +1,12 @@ +diff --git a/src/aws-cpp-sdk-core/include/aws/core/Aws.h b/src/aws-cpp-sdk-core/include/aws/core/Aws.h +index 5c27e75a84c..d221af2039b 100644 +--- a/src/aws-cpp-sdk-core/include/aws/core/Aws.h ++++ b/src/aws-cpp-sdk-core/include/aws/core/Aws.h +@@ -12,6 +12,7 @@ + #include + #include + #include ++#include + #include + #include + diff --git a/dev/vcpkg/ports/aws-sdk-cpp/generateFeatures.ps1 b/dev/vcpkg/ports/aws-sdk-cpp/generateFeatures.ps1 new file mode 100644 index 000000000000..d0f9a5aba722 --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/generateFeatures.ps1 @@ -0,0 +1,73 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)][string]$SourcesRef, + [Parameter(Mandatory=$false)][string]$PortDirectory = $PSScriptRoot, + [Parameter(Mandatory=$false)][string]$vcpkg = "$PSScriptRoot/../../vcpkg" +) + +$ErrorActionPreference = "Stop" + +$ManifestIn = "$PortDirectory/vcpkg.in.json" +$ManifestOut = "$PortDirectory/vcpkg.json" + +$ExtractedSources = "${env:TEMP}/aws-sdk-cpp-generateFeatures-$SourcesRef" +if (-not (Test-Path $ExtractedSources)) { + if (Test-Path "$ExtractedSources.tmp") { + Remove-Item -Force "$ExtractedSources.tmp" + } + git clone "https://github.com/aws/aws-sdk-cpp" "$ExtractedSources.tmp" | Out-Host + git -c "$ExtractedSources.tmp" checkout $SourcesRef + Move-Item "$ExtractedSources.tmp" "$ExtractedSources" +} +Write-Host "Using sources directory: $ExtractedSources" + + +$subfolders = Get-ChildItem -Path "$ExtractedSources\generated\src\aws-cpp-sdk-*", "$ExtractedSources\src\aws-cpp-sdk*" | Sort-Object -Property Name + +$manifest = Get-Content $ManifestIn | ConvertFrom-Json +$manifest | Add-Member ` + -NotePropertyName '$note' ` + -NotePropertyValue 'Automatically generated by generateFeatures.ps1' +$manifest | Add-Member -NotePropertyName 'features' -NotePropertyValue @{} + +function GetDescription($dir, $modulename) +{ + if (Test-Path "$dir\CMakeLists.txt") + { + $descs = @(Select-String -Path "$dir\CMakeLists.txt" -Pattern "`"C\+\+ SDK for the AWS [^`"]*`"") + if ($descs.count -eq 1) { + $desc = $descs[0].Matches.Value -replace "`"","" + "$desc" + } + else { "C++ SDK for the AWS $modulename service" } + } + else { "C++ SDK for the AWS $modulename service" } +} + +$featureDependencies = @{} +Select-String -Path "$ExtractedSources\cmake\sdksCommon.cmake" -Pattern "list\(APPEND SDK_DEPENDENCY_LIST `"([\w-]+):([\w-,]+)`"\)" -AllMatches ` +| ForEach-Object { $_.Matches } ` +| ForEach-Object { $featureDependencies[$_.Groups[1].Value] = @($_.Groups[2].Value -split "," ` +| Where-Object { $_ -ne "core" }) } + +foreach ($subfolder in $subfolders) +{ + $modulename = $subfolder.name -replace "^aws-cpp-sdk-","" + if ($modulename -match "-tests`$") { continue } + if ($modulename -match "-sample`$") { continue } + if ($modulename -eq "core") { continue } + + $lowermodulename = $modulename.ToLower() + + $featureObj = @{ description = (GetDescription $subfolder $modulename) } + + if ($featureDependencies.ContainsKey($lowermodulename)) { + $featureObj.dependencies = ,@{ name = "aws-sdk-cpp"; "default-features" = $false; "features" = $featureDependencies[$lowermodulename] } + } + + $manifest.features.Add("$lowermodulename", $featureObj) +} + +[IO.File]::WriteAllText($ManifestOut, (ConvertTo-Json -Depth 10 -InputObject $manifest)) + +& $vcpkg format-manifest --feature-flags=-manifests $ManifestOut diff --git a/dev/vcpkg/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch b/dev/vcpkg/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch new file mode 100644 index 000000000000..0f7a3b191830 --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch @@ -0,0 +1,20 @@ +diff --git a/src/aws-cpp-sdk-core/CMakeLists.txt b/src/aws-cpp-sdk-core/CMakeLists.txt +index c44546b0e..b66888362 100644 +--- a/src/aws-cpp-sdk-core/CMakeLists.txt ++++ b/src/aws-cpp-sdk-core/CMakeLists.txt +@@ -113,13 +113,8 @@ if(ENABLE_CURL_CLIENT) + int main() { + CURL* handle = curl_easy_init(); + return curl_easy_setopt(handle, CURLOPT_PROXY_SSLCERT, \"client.pem\"); }") +- if (CMAKE_CROSSCOMPILING) +- check_c_source_compiles("${CHECK_CURL_HAS_H2}" CURL_HAS_H2) +- check_c_source_compiles("${CHECK_CURL_HAS_TLS_PROXY}" CURL_HAS_TLS_PROXY) +- else() +- check_c_source_runs("${CHECK_CURL_HAS_H2}" CURL_HAS_H2) +- check_c_source_runs("${CHECK_CURL_HAS_TLS_PROXY}" CURL_HAS_TLS_PROXY) +- endif() ++ set(CURL_HAS_H2 OFF) ++ set(CURL_HAS_TLS_PROXY ON) + elseif(ENABLE_WINDOWS_CLIENT) + # NOTE: HTTP/2 is not supported when using IXML_HTTP_REQUEST_2 + if(USE_IXML_HTTP_REQUEST_2) diff --git a/dev/vcpkg/ports/aws-sdk-cpp/patch-relocatable-rpath.patch b/dev/vcpkg/ports/aws-sdk-cpp/patch-relocatable-rpath.patch new file mode 100644 index 000000000000..1dc1cfd7603d --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/patch-relocatable-rpath.patch @@ -0,0 +1,12 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 41d220d5fa..f6ee9a2a74 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -147,7 +147,6 @@ if (LEGACY_BUILD) + endif () + + # Add Linker search paths to RPATH so as to fix the problem where some linkers can't find cross-compiled dependent libraries in customer paths when linking executables. +- set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) + + # build the sdk targets + project("aws-cpp-sdk-all" VERSION "${PROJECT_VERSION}" LANGUAGES CXX) diff --git a/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake b/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake new file mode 100644 index 000000000000..ac3f2292b8c4 --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake @@ -0,0 +1,101 @@ +vcpkg_buildpath_length_warning(37) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO aws/aws-sdk-cpp + REF "${VERSION}" + SHA512 63de900870e9bec23d42e9458e0e9b1579a9e2dc7b0f404eae1b0dd406898b6d6841c5e2f498710b3828f212705437da3a2fe94813a6c3a842945100a05ae368 + PATCHES + patch-relocatable-rpath.patch + fix-aws-root.patch + lock-curl-http-and-tls-settings.patch + fix-awsmigrationhub-build.patch + fix-header.patch +) + +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) + +set(EXTRA_ARGS) +if(VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + set(rpath "@loader_path") +elseif (VCPKG_TARGET_IS_ANDROID) + set(EXTRA_ARGS "-DTARGET_ARCH=ANDROID" + "-DGIT_EXECUTABLE=--invalid-git-executable--" + "-DGIT_FOUND=TRUE" + "-DNDK_DIR=$ENV{ANDROID_NDK_HOME}" + "-DANDROID_BUILD_ZLIB=FALSE" + "-DANDROID_BUILD_CURL=FALSE" + "-DANDROID_BUILD_OPENSSL=FALSE" + ) +else() + set(rpath "\$ORIGIN") +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + ${EXTRA_ARGS} + "-DENABLE_UNITY_BUILD=ON" + "-DENABLE_TESTING=OFF" + "-DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}" + "-DBUILD_ONLY=${FEATURES}" + "-DBUILD_DEPS=OFF" + "-DBUILD_SHARED_LIBS=OFF" + "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" + "-DCMAKE_INSTALL_RPATH=${rpath}" + "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files +) +vcpkg_cmake_install() + +foreach(TARGET IN LISTS FEATURES) + vcpkg_cmake_config_fixup(PACKAGE_NAME "aws-cpp-sdk-${TARGET}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) +endforeach() +vcpkg_cmake_config_fixup(PACKAGE_NAME "AWSSDK" CONFIG_PATH "lib/cmake/AWSSDK") + +vcpkg_copy_pdbs() + +file(GLOB_RECURSE AWS_TARGETS "${CURRENT_PACKAGES_DIR}/share/*/*-targets-*.cmake") +foreach(AWS_TARGET IN LISTS AWS_TARGETS) + file(READ ${AWS_TARGET} _contents) + string(REGEX REPLACE + "bin\\/([A-Za-z0-9_.-]+\\.lib)" + "lib/\\1" + _contents "${_contents}") + file(WRITE ${AWS_TARGET} "${_contents}") +endforeach() + +file(GLOB AWS_CONFIGS "${CURRENT_PACKAGES_DIR}/share/*/aws-cpp-sdk-*-config.cmake") +list(FILTER AWS_CONFIGS EXCLUDE REGEX "aws-cpp-sdk-core-config\\.cmake\$") +foreach(AWS_CONFIG IN LISTS AWS_CONFIGS) + file(READ "${AWS_CONFIG}" _contents) + file(WRITE "${AWS_CONFIG}" "include(CMakeFindDependencyMacro)\nfind_dependency(aws-cpp-sdk-core)\n${_contents}") +endforeach() + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/lib/pkgconfig" + "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig" + "${CURRENT_PACKAGES_DIR}/nuget" + "${CURRENT_PACKAGES_DIR}/debug/nuget" +) + +if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + file(GLOB LIB_FILES ${CURRENT_PACKAGES_DIR}/bin/*.lib) + if(LIB_FILES) + file(COPY ${LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(REMOVE ${LIB_FILES}) + endif() + file(GLOB DEBUG_LIB_FILES ${CURRENT_PACKAGES_DIR}/debug/bin/*.lib) + if(DEBUG_LIB_FILES) + file(COPY ${DEBUG_LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) + file(REMOVE ${DEBUG_LIB_FILES}) + endif() + + file(APPEND "${CURRENT_PACKAGES_DIR}/include/aws/core/SDKConfig.h" "#ifndef USE_IMPORT_EXPORT\n#define USE_IMPORT_EXPORT\n#endif") +endif() + +configure_file("${CURRENT_PORT_DIR}/usage" "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" @ONLY) + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/dev/vcpkg/ports/aws-sdk-cpp/usage b/dev/vcpkg/ports/aws-sdk-cpp/usage new file mode 100644 index 000000000000..37e1f617b2cb --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/usage @@ -0,0 +1,12 @@ +The package @PORT@:@TARGET_TRIPLET@ provides CMake targets: + + When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. + find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) + target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) + target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) + + OR + + find_package(aws-cpp-sdk-core REQUIRED) + target_include_directories(main PRIVATE aws-cpp-sdk-core) + target_link_libraries(main PRIVATE aws-cpp-sdk-core) diff --git a/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json new file mode 100644 index 000000000000..a618a77d864b --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json @@ -0,0 +1,38 @@ +{ + "name": "aws-sdk-cpp", + "version": "1.11.160", + "port-version": 1, + "description": "AWS SDK for C++", + "homepage": "https://github.com/aws/aws-sdk-cpp", + "license": "Apache-2.0", + "supports": "!(windows & arm) & !uwp", + "dependencies": [ + "aws-crt-cpp", + { + "name": "curl", + "default-features": false, + "features": [ + "ssl" + ], + "platform": "!uwp & !windows" + }, + { + "name": "openssl", + "platform": "!uwp & !windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "zlib" + ], + "default-features": [ + "dynamodb", + "kinesis", + "s3" + ] +} diff --git a/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json new file mode 100644 index 000000000000..138894a9eec9 --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json @@ -0,0 +1,1187 @@ +{ + "$note": "Automatically generated by generateFeatures.ps1", + "name": "aws-sdk-cpp", + "version": "1.11.169", + "port-version": 2, + "description": "AWS SDK for C++", + "homepage": "https://github.com/aws/aws-sdk-cpp", + "license": "Apache-2.0", + "supports": "!(windows & arm) & !uwp", + "dependencies": [ + "aws-crt-cpp", + { + "name": "curl", + "default-features": false, + "features": [ + "ssl" + ], + "platform": "!uwp & !windows" + }, + { + "name": "openssl", + "platform": "!uwp & !windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "zlib" + ], + "default-features": [ + "dynamodb", + "kinesis", + "s3" + ], + "features": { + "access-management": { + "description": "C++ SDK for the AWS access-management service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "cognito-identity", + "iam" + ] + } + ] + }, + "accessanalyzer": { + "description": "C++ SDK for the AWS accessanalyzer service" + }, + "account": { + "description": "C++ SDK for the AWS account service" + }, + "acm": { + "description": "C++ SDK for the AWS acm service" + }, + "acm-pca": { + "description": "C++ SDK for the AWS acm-pca service" + }, + "alexaforbusiness": { + "description": "C++ SDK for the AWS alexaforbusiness service" + }, + "amp": { + "description": "C++ SDK for the AWS amp service" + }, + "amplify": { + "description": "C++ SDK for the AWS amplify service" + }, + "amplifybackend": { + "description": "C++ SDK for the AWS amplifybackend service" + }, + "amplifyuibuilder": { + "description": "C++ SDK for the AWS amplifyuibuilder service" + }, + "apigateway": { + "description": "C++ SDK for the AWS apigateway service" + }, + "apigatewaymanagementapi": { + "description": "C++ SDK for the AWS apigatewaymanagementapi service" + }, + "apigatewayv2": { + "description": "C++ SDK for the AWS apigatewayv2 service" + }, + "appconfig": { + "description": "C++ SDK for the AWS appconfig service" + }, + "appconfigdata": { + "description": "C++ SDK for the AWS appconfigdata service" + }, + "appfabric": { + "description": "C++ SDK for the AWS appfabric service" + }, + "appflow": { + "description": "C++ SDK for the AWS appflow service" + }, + "appintegrations": { + "description": "C++ SDK for the AWS appintegrations service" + }, + "application-autoscaling": { + "description": "C++ SDK for the AWS application-autoscaling service" + }, + "application-insights": { + "description": "C++ SDK for the AWS application-insights service" + }, + "applicationcostprofiler": { + "description": "C++ SDK for the AWS applicationcostprofiler service" + }, + "appmesh": { + "description": "C++ SDK for the AWS appmesh service" + }, + "apprunner": { + "description": "C++ SDK for the AWS apprunner service" + }, + "appstream": { + "description": "C++ SDK for the AWS appstream service" + }, + "appsync": { + "description": "C++ SDK for the AWS appsync service" + }, + "arc-zonal-shift": { + "description": "C++ SDK for the AWS arc-zonal-shift service" + }, + "athena": { + "description": "C++ SDK for the AWS athena service" + }, + "auditmanager": { + "description": "C++ SDK for the AWS auditmanager service" + }, + "autoscaling": { + "description": "C++ SDK for the AWS autoscaling service" + }, + "autoscaling-plans": { + "description": "C++ SDK for the AWS autoscaling-plans service" + }, + "awsmigrationhub": { + "description": "C++ SDK for the AWS AWSMigrationHub service" + }, + "awstransfer": { + "description": "C++ SDK for the AWS awstransfer service" + }, + "backup": { + "description": "C++ SDK for the AWS backup service" + }, + "backup-gateway": { + "description": "C++ SDK for the AWS backup-gateway service" + }, + "backupstorage": { + "description": "C++ SDK for the AWS backupstorage service" + }, + "batch": { + "description": "C++ SDK for the AWS batch service" + }, + "billingconductor": { + "description": "C++ SDK for the AWS billingconductor service" + }, + "braket": { + "description": "C++ SDK for the AWS braket service" + }, + "budgets": { + "description": "C++ SDK for the AWS budgets service" + }, + "ce": { + "description": "C++ SDK for the AWS ce service" + }, + "chime": { + "description": "C++ SDK for the AWS chime service" + }, + "chime-sdk-identity": { + "description": "C++ SDK for the AWS chime-sdk-identity service" + }, + "chime-sdk-media-pipelines": { + "description": "C++ SDK for the AWS chime-sdk-media-pipelines service" + }, + "chime-sdk-meetings": { + "description": "C++ SDK for the AWS chime-sdk-meetings service" + }, + "chime-sdk-messaging": { + "description": "C++ SDK for the AWS chime-sdk-messaging service" + }, + "chime-sdk-voice": { + "description": "C++ SDK for the AWS chime-sdk-voice service" + }, + "cleanrooms": { + "description": "C++ SDK for the AWS cleanrooms service" + }, + "cloud9": { + "description": "C++ SDK for the AWS cloud9 service" + }, + "cloudcontrol": { + "description": "C++ SDK for the AWS cloudcontrol service" + }, + "clouddirectory": { + "description": "C++ SDK for the AWS clouddirectory service" + }, + "cloudformation": { + "description": "C++ SDK for the AWS cloudformation service" + }, + "cloudfront": { + "description": "C++ SDK for the AWS cloudfront service" + }, + "cloudhsm": { + "description": "C++ SDK for the AWS cloudhsm service" + }, + "cloudhsmv2": { + "description": "C++ SDK for the AWS cloudhsmv2 service" + }, + "cloudsearch": { + "description": "C++ SDK for the AWS cloudsearch service" + }, + "cloudsearchdomain": { + "description": "C++ SDK for the AWS cloudsearchdomain service" + }, + "cloudtrail": { + "description": "C++ SDK for the AWS cloudtrail service" + }, + "cloudtrail-data": { + "description": "C++ SDK for the AWS cloudtrail-data service" + }, + "codeartifact": { + "description": "C++ SDK for the AWS codeartifact service" + }, + "codebuild": { + "description": "C++ SDK for the AWS codebuild service" + }, + "codecatalyst": { + "description": "C++ SDK for the AWS codecatalyst service" + }, + "codecommit": { + "description": "C++ SDK for the AWS codecommit service" + }, + "codedeploy": { + "description": "C++ SDK for the AWS codedeploy service" + }, + "codeguru-reviewer": { + "description": "C++ SDK for the AWS codeguru-reviewer service" + }, + "codeguru-security": { + "description": "C++ SDK for the AWS codeguru-security service" + }, + "codeguruprofiler": { + "description": "C++ SDK for the AWS codeguruprofiler service" + }, + "codepipeline": { + "description": "C++ SDK for the AWS codepipeline service" + }, + "codestar": { + "description": "C++ SDK for the AWS codestar service" + }, + "codestar-connections": { + "description": "C++ SDK for the AWS codestar-connections service" + }, + "codestar-notifications": { + "description": "C++ SDK for the AWS codestar-notifications service" + }, + "cognito-identity": { + "description": "C++ SDK for the AWS cognito-identity service" + }, + "cognito-idp": { + "description": "C++ SDK for the AWS cognito-idp service" + }, + "cognito-sync": { + "description": "C++ SDK for the AWS cognito-sync service" + }, + "comprehend": { + "description": "C++ SDK for the AWS comprehend service" + }, + "comprehendmedical": { + "description": "C++ SDK for the AWS comprehendmedical service" + }, + "compute-optimizer": { + "description": "C++ SDK for the AWS compute-optimizer service" + }, + "config": { + "description": "C++ SDK for the AWS config service" + }, + "connect": { + "description": "C++ SDK for the AWS connect service" + }, + "connect-contact-lens": { + "description": "C++ SDK for the AWS connect-contact-lens service" + }, + "connectcampaigns": { + "description": "C++ SDK for the AWS connectcampaigns service" + }, + "connectcases": { + "description": "C++ SDK for the AWS connectcases service" + }, + "connectparticipant": { + "description": "C++ SDK for the AWS connectparticipant service" + }, + "controltower": { + "description": "C++ SDK for the AWS controltower service" + }, + "cur": { + "description": "C++ SDK for the AWS cur service" + }, + "customer-profiles": { + "description": "C++ SDK for the AWS customer-profiles service" + }, + "databrew": { + "description": "C++ SDK for the AWS databrew service" + }, + "dataexchange": { + "description": "C++ SDK for the AWS dataexchange service" + }, + "datapipeline": { + "description": "C++ SDK for the AWS datapipeline service" + }, + "datasync": { + "description": "C++ SDK for the AWS datasync service" + }, + "dax": { + "description": "C++ SDK for the AWS dax service" + }, + "detective": { + "description": "C++ SDK for the AWS detective service" + }, + "devicefarm": { + "description": "C++ SDK for the AWS devicefarm service" + }, + "devops-guru": { + "description": "C++ SDK for the AWS devops-guru service" + }, + "directconnect": { + "description": "C++ SDK for the AWS directconnect service" + }, + "discovery": { + "description": "C++ SDK for the AWS discovery service" + }, + "dlm": { + "description": "C++ SDK for the AWS dlm service" + }, + "dms": { + "description": "C++ SDK for the AWS dms service" + }, + "docdb": { + "description": "C++ SDK for the AWS docdb service" + }, + "docdb-elastic": { + "description": "C++ SDK for the AWS docdb-elastic service" + }, + "drs": { + "description": "C++ SDK for the AWS drs service" + }, + "ds": { + "description": "C++ SDK for the AWS ds service" + }, + "dynamodb": { + "description": "C++ SDK for the AWS dynamodb service" + }, + "dynamodbstreams": { + "description": "C++ SDK for the AWS dynamodbstreams service" + }, + "ebs": { + "description": "C++ SDK for the AWS ebs service" + }, + "ec2": { + "description": "C++ SDK for the AWS ec2 service" + }, + "ec2-instance-connect": { + "description": "C++ SDK for the AWS ec2-instance-connect service" + }, + "ecr": { + "description": "C++ SDK for the AWS ecr service" + }, + "ecr-public": { + "description": "C++ SDK for the AWS ecr-public service" + }, + "ecs": { + "description": "C++ SDK for the AWS ecs service" + }, + "eks": { + "description": "C++ SDK for the AWS eks service" + }, + "elastic-inference": { + "description": "C++ SDK for the AWS elastic-inference service" + }, + "elasticache": { + "description": "C++ SDK for the AWS elasticache service" + }, + "elasticbeanstalk": { + "description": "C++ SDK for the AWS elasticbeanstalk service" + }, + "elasticfilesystem": { + "description": "C++ SDK for the AWS elasticfilesystem service" + }, + "elasticloadbalancing": { + "description": "C++ SDK for the AWS elasticloadbalancing service" + }, + "elasticloadbalancingv2": { + "description": "C++ SDK for the AWS elasticloadbalancingv2 service" + }, + "elasticmapreduce": { + "description": "C++ SDK for the AWS elasticmapreduce service" + }, + "elastictranscoder": { + "description": "C++ SDK for the AWS elastictranscoder service" + }, + "email": { + "description": "C++ SDK for the AWS email service" + }, + "emr-containers": { + "description": "C++ SDK for the AWS emr-containers service" + }, + "emr-serverless": { + "description": "C++ SDK for the AWS emr-serverless service" + }, + "entityresolution": { + "description": "C++ SDK for the AWS entityresolution service" + }, + "es": { + "description": "C++ SDK for the AWS es service" + }, + "eventbridge": { + "description": "C++ SDK for the AWS eventbridge service" + }, + "events": { + "description": "C++ SDK for the AWS events service" + }, + "evidently": { + "description": "C++ SDK for the AWS evidently service" + }, + "finspace": { + "description": "C++ SDK for the AWS finspace service" + }, + "finspace-data": { + "description": "C++ SDK for the AWS finspace-data service" + }, + "firehose": { + "description": "C++ SDK for the AWS firehose service" + }, + "fis": { + "description": "C++ SDK for the AWS fis service" + }, + "fms": { + "description": "C++ SDK for the AWS fms service" + }, + "forecast": { + "description": "C++ SDK for the AWS forecast service" + }, + "forecastquery": { + "description": "C++ SDK for the AWS forecastquery service" + }, + "frauddetector": { + "description": "C++ SDK for the AWS frauddetector service" + }, + "fsx": { + "description": "C++ SDK for the AWS fsx service" + }, + "gamelift": { + "description": "C++ SDK for the AWS gamelift service" + }, + "gamesparks": { + "description": "C++ SDK for the AWS gamesparks service" + }, + "glacier": { + "description": "C++ SDK for the AWS glacier service" + }, + "globalaccelerator": { + "description": "C++ SDK for the AWS globalaccelerator service" + }, + "glue": { + "description": "C++ SDK for the AWS glue service" + }, + "grafana": { + "description": "C++ SDK for the AWS grafana service" + }, + "greengrass": { + "description": "C++ SDK for the AWS greengrass service" + }, + "greengrassv2": { + "description": "C++ SDK for the AWS greengrassv2 service" + }, + "groundstation": { + "description": "C++ SDK for the AWS groundstation service" + }, + "guardduty": { + "description": "C++ SDK for the AWS guardduty service" + }, + "health": { + "description": "C++ SDK for the AWS health service" + }, + "healthlake": { + "description": "C++ SDK for the AWS healthlake service" + }, + "honeycode": { + "description": "C++ SDK for the AWS honeycode service" + }, + "iam": { + "description": "C++ SDK for the AWS iam service" + }, + "identity-management": { + "description": "C++ SDK for the AWS identity-management service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "cognito-identity", + "sts" + ] + } + ] + }, + "identitystore": { + "description": "C++ SDK for the AWS identitystore service" + }, + "imagebuilder": { + "description": "C++ SDK for the AWS imagebuilder service" + }, + "importexport": { + "description": "C++ SDK for the AWS importexport service" + }, + "inspector": { + "description": "C++ SDK for the AWS inspector service" + }, + "inspector2": { + "description": "C++ SDK for the AWS inspector2 service" + }, + "internetmonitor": { + "description": "C++ SDK for the AWS internetmonitor service" + }, + "iot": { + "description": "C++ SDK for the AWS iot service" + }, + "iot-data": { + "description": "C++ SDK for the AWS iot-data service" + }, + "iot-jobs-data": { + "description": "C++ SDK for the AWS iot-jobs-data service" + }, + "iot-roborunner": { + "description": "C++ SDK for the AWS iot-roborunner service" + }, + "iot1click-devices": { + "description": "C++ SDK for the AWS iot1click-devices service" + }, + "iot1click-projects": { + "description": "C++ SDK for the AWS iot1click-projects service" + }, + "iotanalytics": { + "description": "C++ SDK for the AWS iotanalytics service" + }, + "iotdeviceadvisor": { + "description": "C++ SDK for the AWS iotdeviceadvisor service" + }, + "iotevents": { + "description": "C++ SDK for the AWS iotevents service" + }, + "iotevents-data": { + "description": "C++ SDK for the AWS iotevents-data service" + }, + "iotfleethub": { + "description": "C++ SDK for the AWS iotfleethub service" + }, + "iotfleetwise": { + "description": "C++ SDK for the AWS iotfleetwise service" + }, + "iotsecuretunneling": { + "description": "C++ SDK for the AWS iotsecuretunneling service" + }, + "iotsitewise": { + "description": "C++ SDK for the AWS iotsitewise service" + }, + "iotthingsgraph": { + "description": "C++ SDK for the AWS iotthingsgraph service" + }, + "iottwinmaker": { + "description": "C++ SDK for the AWS iottwinmaker service" + }, + "iotwireless": { + "description": "C++ SDK for the AWS iotwireless service" + }, + "ivs": { + "description": "C++ SDK for the AWS ivs service" + }, + "ivs-realtime": { + "description": "C++ SDK for the AWS ivs-realtime service" + }, + "ivschat": { + "description": "C++ SDK for the AWS ivschat service" + }, + "kafka": { + "description": "C++ SDK for the AWS kafka service" + }, + "kafkaconnect": { + "description": "C++ SDK for the AWS kafkaconnect service" + }, + "kendra": { + "description": "C++ SDK for the AWS kendra service" + }, + "kendra-ranking": { + "description": "C++ SDK for the AWS kendra-ranking service" + }, + "keyspaces": { + "description": "C++ SDK for the AWS keyspaces service" + }, + "kinesis": { + "description": "C++ SDK for the AWS kinesis service" + }, + "kinesis-video-archived-media": { + "description": "C++ SDK for the AWS kinesis-video-archived-media service" + }, + "kinesis-video-media": { + "description": "C++ SDK for the AWS kinesis-video-media service" + }, + "kinesis-video-signaling": { + "description": "C++ SDK for the AWS kinesis-video-signaling service" + }, + "kinesis-video-webrtc-storage": { + "description": "C++ SDK for the AWS kinesis-video-webrtc-storage service" + }, + "kinesisanalytics": { + "description": "C++ SDK for the AWS kinesisanalytics service" + }, + "kinesisanalyticsv2": { + "description": "C++ SDK for the AWS kinesisanalyticsv2 service" + }, + "kinesisvideo": { + "description": "C++ SDK for the AWS kinesisvideo service" + }, + "kms": { + "description": "C++ SDK for the AWS kms service" + }, + "lakeformation": { + "description": "C++ SDK for the AWS lakeformation service" + }, + "lambda": { + "description": "C++ SDK for the AWS lambda service" + }, + "lex": { + "description": "C++ SDK for the AWS lex service" + }, + "lex-models": { + "description": "C++ SDK for the AWS lex-models service" + }, + "lexv2-models": { + "description": "C++ SDK for the AWS lexv2-models service" + }, + "lexv2-runtime": { + "description": "C++ SDK for the AWS lexv2-runtime service" + }, + "license-manager": { + "description": "C++ SDK for the AWS license-manager service" + }, + "license-manager-linux-subscriptions": { + "description": "C++ SDK for the AWS license-manager-linux-subscriptions service" + }, + "license-manager-user-subscriptions": { + "description": "C++ SDK for the AWS license-manager-user-subscriptions service" + }, + "lightsail": { + "description": "C++ SDK for the AWS lightsail service" + }, + "location": { + "description": "C++ SDK for the AWS location service" + }, + "logs": { + "description": "C++ SDK for the AWS logs service" + }, + "lookoutequipment": { + "description": "C++ SDK for the AWS lookoutequipment service" + }, + "lookoutmetrics": { + "description": "C++ SDK for the AWS lookoutmetrics service" + }, + "lookoutvision": { + "description": "C++ SDK for the AWS lookoutvision service" + }, + "m2": { + "description": "C++ SDK for the AWS m2 service" + }, + "machinelearning": { + "description": "C++ SDK for the AWS machinelearning service" + }, + "macie": { + "description": "C++ SDK for the AWS macie service" + }, + "macie2": { + "description": "C++ SDK for the AWS macie2 service" + }, + "managedblockchain": { + "description": "C++ SDK for the AWS managedblockchain service" + }, + "managedblockchain-query": { + "description": "C++ SDK for the AWS managedblockchain-query service" + }, + "marketplace-catalog": { + "description": "C++ SDK for the AWS marketplace-catalog service" + }, + "marketplace-entitlement": { + "description": "C++ SDK for the AWS marketplace-entitlement service" + }, + "marketplacecommerceanalytics": { + "description": "C++ SDK for the AWS marketplacecommerceanalytics service" + }, + "mediaconnect": { + "description": "C++ SDK for the AWS mediaconnect service" + }, + "mediaconvert": { + "description": "C++ SDK for the AWS mediaconvert service" + }, + "medialive": { + "description": "C++ SDK for the AWS medialive service" + }, + "mediapackage": { + "description": "C++ SDK for the AWS mediapackage service" + }, + "mediapackage-vod": { + "description": "C++ SDK for the AWS mediapackage-vod service" + }, + "mediapackagev2": { + "description": "C++ SDK for the AWS mediapackagev2 service" + }, + "mediastore": { + "description": "C++ SDK for the AWS mediastore service" + }, + "mediastore-data": { + "description": "C++ SDK for the AWS mediastore-data service" + }, + "mediatailor": { + "description": "C++ SDK for the AWS mediatailor service" + }, + "medical-imaging": { + "description": "C++ SDK for the AWS medical-imaging service" + }, + "memorydb": { + "description": "C++ SDK for the AWS memorydb service" + }, + "meteringmarketplace": { + "description": "C++ SDK for the AWS meteringmarketplace service" + }, + "mgn": { + "description": "C++ SDK for the AWS mgn service" + }, + "migration-hub-refactor-spaces": { + "description": "C++ SDK for the AWS migration-hub-refactor-spaces service" + }, + "migrationhub-config": { + "description": "C++ SDK for the AWS migrationhub-config service" + }, + "migrationhuborchestrator": { + "description": "C++ SDK for the AWS migrationhuborchestrator service" + }, + "migrationhubstrategy": { + "description": "C++ SDK for the AWS migrationhubstrategy service" + }, + "mobile": { + "description": "C++ SDK for the AWS mobile service" + }, + "monitoring": { + "description": "C++ SDK for the AWS monitoring service" + }, + "mq": { + "description": "C++ SDK for the AWS mq service" + }, + "mturk-requester": { + "description": "C++ SDK for the AWS mturk-requester service" + }, + "mwaa": { + "description": "C++ SDK for the AWS mwaa service" + }, + "neptune": { + "description": "C++ SDK for the AWS neptune service" + }, + "neptunedata": { + "description": "C++ SDK for the AWS neptunedata service" + }, + "network-firewall": { + "description": "C++ SDK for the AWS network-firewall service" + }, + "networkmanager": { + "description": "C++ SDK for the AWS networkmanager service" + }, + "nimble": { + "description": "C++ SDK for the AWS nimble service" + }, + "oam": { + "description": "C++ SDK for the AWS oam service" + }, + "omics": { + "description": "C++ SDK for the AWS omics service" + }, + "opensearch": { + "description": "C++ SDK for the AWS opensearch service" + }, + "opensearchserverless": { + "description": "C++ SDK for the AWS opensearchserverless service" + }, + "opsworks": { + "description": "C++ SDK for the AWS opsworks service" + }, + "opsworkscm": { + "description": "C++ SDK for the AWS opsworkscm service" + }, + "organizations": { + "description": "C++ SDK for the AWS organizations service" + }, + "osis": { + "description": "C++ SDK for the AWS osis service" + }, + "outposts": { + "description": "C++ SDK for the AWS outposts service" + }, + "panorama": { + "description": "C++ SDK for the AWS panorama service" + }, + "payment-cryptography": { + "description": "C++ SDK for the AWS payment-cryptography service" + }, + "payment-cryptography-data": { + "description": "C++ SDK for the AWS payment-cryptography-data service" + }, + "pca-connector-ad": { + "description": "C++ SDK for the AWS pca-connector-ad service" + }, + "personalize": { + "description": "C++ SDK for the AWS personalize service" + }, + "personalize-events": { + "description": "C++ SDK for the AWS personalize-events service" + }, + "personalize-runtime": { + "description": "C++ SDK for the AWS personalize-runtime service" + }, + "pi": { + "description": "C++ SDK for the AWS pi service" + }, + "pinpoint": { + "description": "C++ SDK for the AWS pinpoint service" + }, + "pinpoint-email": { + "description": "C++ SDK for the AWS pinpoint-email service" + }, + "pinpoint-sms-voice-v2": { + "description": "C++ SDK for the AWS pinpoint-sms-voice-v2 service" + }, + "pipes": { + "description": "C++ SDK for the AWS pipes service" + }, + "polly": { + "description": "C++ SDK for the AWS polly service" + }, + "pricing": { + "description": "C++ SDK for the AWS pricing service" + }, + "privatenetworks": { + "description": "C++ SDK for the AWS privatenetworks service" + }, + "proton": { + "description": "C++ SDK for the AWS proton service" + }, + "qldb": { + "description": "C++ SDK for the AWS qldb service" + }, + "qldb-session": { + "description": "C++ SDK for the AWS qldb-session service" + }, + "queues": { + "description": "C++ SDK for the AWS queues service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "sqs" + ] + } + ] + }, + "quicksight": { + "description": "C++ SDK for the AWS quicksight service" + }, + "ram": { + "description": "C++ SDK for the AWS ram service" + }, + "rbin": { + "description": "C++ SDK for the AWS rbin service" + }, + "rds": { + "description": "C++ SDK for the AWS rds service" + }, + "rds-data": { + "description": "C++ SDK for the AWS rds-data service" + }, + "redshift": { + "description": "C++ SDK for the AWS redshift service" + }, + "redshift-data": { + "description": "C++ SDK for the AWS redshift-data service" + }, + "redshift-serverless": { + "description": "C++ SDK for the AWS redshift-serverless service" + }, + "rekognition": { + "description": "C++ SDK for the AWS rekognition service" + }, + "resiliencehub": { + "description": "C++ SDK for the AWS resiliencehub service" + }, + "resource-explorer-2": { + "description": "C++ SDK for the AWS resource-explorer-2 service" + }, + "resource-groups": { + "description": "C++ SDK for the AWS resource-groups service" + }, + "resourcegroupstaggingapi": { + "description": "C++ SDK for the AWS resourcegroupstaggingapi service" + }, + "robomaker": { + "description": "C++ SDK for the AWS robomaker service" + }, + "rolesanywhere": { + "description": "C++ SDK for the AWS rolesanywhere service" + }, + "route53": { + "description": "C++ SDK for the AWS route53 service" + }, + "route53-recovery-cluster": { + "description": "C++ SDK for the AWS route53-recovery-cluster service" + }, + "route53-recovery-control-config": { + "description": "C++ SDK for the AWS route53-recovery-control-config service" + }, + "route53-recovery-readiness": { + "description": "C++ SDK for the AWS route53-recovery-readiness service" + }, + "route53domains": { + "description": "C++ SDK for the AWS route53domains service" + }, + "route53resolver": { + "description": "C++ SDK for the AWS route53resolver service" + }, + "rum": { + "description": "C++ SDK for the AWS rum service" + }, + "s3": { + "description": "C++ SDK for the AWS s3 service" + }, + "s3-crt": { + "description": "C++ SDK for the AWS s3-crt service" + }, + "s3-encryption": { + "description": "C++ SDK for the AWS s3-encryption service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "kms", + "s3" + ] + } + ] + }, + "s3control": { + "description": "C++ SDK for the AWS s3control service" + }, + "s3outposts": { + "description": "C++ SDK for the AWS s3outposts service" + }, + "sagemaker": { + "description": "C++ SDK for the AWS sagemaker service" + }, + "sagemaker-a2i-runtime": { + "description": "C++ SDK for the AWS sagemaker-a2i-runtime service" + }, + "sagemaker-edge": { + "description": "C++ SDK for the AWS sagemaker-edge service" + }, + "sagemaker-featurestore-runtime": { + "description": "C++ SDK for the AWS sagemaker-featurestore-runtime service" + }, + "sagemaker-geospatial": { + "description": "C++ SDK for the AWS sagemaker-geospatial service" + }, + "sagemaker-metrics": { + "description": "C++ SDK for the AWS sagemaker-metrics service" + }, + "sagemaker-runtime": { + "description": "C++ SDK for the AWS sagemaker-runtime service" + }, + "savingsplans": { + "description": "C++ SDK for the AWS savingsplans service" + }, + "scheduler": { + "description": "C++ SDK for the AWS scheduler service" + }, + "schemas": { + "description": "C++ SDK for the AWS schemas service" + }, + "sdb": { + "description": "C++ SDK for the AWS sdb service" + }, + "secretsmanager": { + "description": "C++ SDK for the AWS secretsmanager service" + }, + "securityhub": { + "description": "C++ SDK for the AWS securityhub service" + }, + "securitylake": { + "description": "C++ SDK for the AWS securitylake service" + }, + "serverlessrepo": { + "description": "C++ SDK for the AWS serverlessrepo service" + }, + "service-quotas": { + "description": "C++ SDK for the AWS service-quotas service" + }, + "servicecatalog": { + "description": "C++ SDK for the AWS servicecatalog service" + }, + "servicecatalog-appregistry": { + "description": "C++ SDK for the AWS servicecatalog-appregistry service" + }, + "servicediscovery": { + "description": "C++ SDK for the AWS servicediscovery service" + }, + "sesv2": { + "description": "C++ SDK for the AWS sesv2 service" + }, + "shield": { + "description": "C++ SDK for the AWS shield service" + }, + "signer": { + "description": "C++ SDK for the AWS signer service" + }, + "simspaceweaver": { + "description": "C++ SDK for the AWS simspaceweaver service" + }, + "sms": { + "description": "C++ SDK for the AWS sms service" + }, + "sms-voice": { + "description": "C++ SDK for the AWS sms-voice service" + }, + "snow-device-management": { + "description": "C++ SDK for the AWS snow-device-management service" + }, + "snowball": { + "description": "C++ SDK for the AWS snowball service" + }, + "sns": { + "description": "C++ SDK for the AWS sns service" + }, + "sqs": { + "description": "C++ SDK for the AWS sqs service" + }, + "ssm": { + "description": "C++ SDK for the AWS ssm service" + }, + "ssm-contacts": { + "description": "C++ SDK for the AWS ssm-contacts service" + }, + "ssm-incidents": { + "description": "C++ SDK for the AWS ssm-incidents service" + }, + "ssm-sap": { + "description": "C++ SDK for the AWS ssm-sap service" + }, + "sso": { + "description": "C++ SDK for the AWS sso service" + }, + "sso-admin": { + "description": "C++ SDK for the AWS sso-admin service" + }, + "sso-oidc": { + "description": "C++ SDK for the AWS sso-oidc service" + }, + "states": { + "description": "C++ SDK for the AWS states service" + }, + "storagegateway": { + "description": "C++ SDK for the AWS storagegateway service" + }, + "sts": { + "description": "C++ SDK for the AWS sts service" + }, + "support": { + "description": "C++ SDK for the AWS support service" + }, + "support-app": { + "description": "C++ SDK for the AWS support-app service" + }, + "swf": { + "description": "C++ SDK for the AWS swf service" + }, + "synthetics": { + "description": "C++ SDK for the AWS synthetics service" + }, + "text-to-speech": { + "description": "C++ SDK for the AWS text-to-speech service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "polly" + ] + } + ] + }, + "textract": { + "description": "C++ SDK for the AWS textract service" + }, + "timestream-query": { + "description": "C++ SDK for the AWS timestream-query service" + }, + "timestream-write": { + "description": "C++ SDK for the AWS timestream-write service" + }, + "tnb": { + "description": "C++ SDK for the AWS tnb service" + }, + "transcribe": { + "description": "C++ SDK for the AWS transcribe service" + }, + "transcribestreaming": { + "description": "C++ SDK for the AWS transcribestreaming service" + }, + "transfer": { + "description": "C++ SDK for the AWS transfer service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "s3" + ] + } + ] + }, + "translate": { + "description": "C++ SDK for the AWS translate service" + }, + "verifiedpermissions": { + "description": "C++ SDK for the AWS verifiedpermissions service" + }, + "voice-id": { + "description": "C++ SDK for the AWS voice-id service" + }, + "vpc-lattice": { + "description": "C++ SDK for the AWS vpc-lattice service" + }, + "waf": { + "description": "C++ SDK for the AWS waf service" + }, + "waf-regional": { + "description": "C++ SDK for the AWS waf-regional service" + }, + "wafv2": { + "description": "C++ SDK for the AWS wafv2 service" + }, + "wellarchitected": { + "description": "C++ SDK for the AWS wellarchitected service" + }, + "wisdom": { + "description": "C++ SDK for the AWS wisdom service" + }, + "workdocs": { + "description": "C++ SDK for the AWS workdocs service" + }, + "worklink": { + "description": "C++ SDK for the AWS worklink service" + }, + "workmail": { + "description": "C++ SDK for the AWS workmail service" + }, + "workmailmessageflow": { + "description": "C++ SDK for the AWS workmailmessageflow service" + }, + "workspaces": { + "description": "C++ SDK for the AWS workspaces service" + }, + "workspaces-web": { + "description": "C++ SDK for the AWS workspaces-web service" + }, + "xray": { + "description": "C++ SDK for the AWS xray service" + } + } +} diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index f11d4d409552..4593c86e7d51 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -60,7 +60,6 @@ "dependencies": [ { "name": "aws-sdk-cpp", - "version>=": "1.11.169", "features": [ "identity-management", "s3", diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index df78634c72ad..90c2ed119914 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -31,7 +31,7 @@ Please set them via `--`, e.g. `--build_type=Release`. | velox_home | Specify your own Velox source path to build. | "" | | build_velox_tests | Build Velox tests. | OFF | | build_velox_benchmarks | Build Velox benchmarks (velox_tests and connectors will be disabled if ON) | OFF | -| compile_arrow_java | Compile arrow java for gluten build to use to fix invalid pointer issues. | OFF | +| compile_arrow_java | Compile arrow java for gluten build to use to fix invalid pointer issues. | ON | ### Velox build parameters for build_velox.sh Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`. @@ -49,7 +49,7 @@ Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`. | build_test_utils | Build Velox with cmake arg -DVELOX_BUILD_TEST_UTILS=ON if ON. | OFF | | build_tests | Build Velox test. | OFF | | build_benchmarks | Build Velox benchmarks. | OFF | -| compile_arrow_java | Build arrow java for gluten build to use to fix invalid pointer issues. | OFF | +| compile_arrow_java | Build arrow java for gluten build to use to fix invalid pointer issues. | ON | ### Maven build parameters The below parameters can be set via `-P` for mvn. diff --git a/docs/velox-backend-troubleshooting.md b/docs/velox-backend-troubleshooting.md index 32289ab230bc..98f82461f351 100644 --- a/docs/velox-backend-troubleshooting.md +++ b/docs/velox-backend-troubleshooting.md @@ -32,7 +32,9 @@ rm -rf $SPARK_HOME/jars/arrow-* ``` ### Incompatible class error when using native writer -Gluten native writer overwrite some vanilla spark classes. Therefore, when running a program that uses gluten, it is essential to ensure that the gluten jar is loaded prior to the vanilla spark jar. In this section, we will provide some configuration settings in `$SPARK_HOME/conf/spark-defaults.conf` for Yarn client, Yarn cluster, and Local&Standalone mode to guarantee that the gluten jar is prioritized. +Gluten native writer overwrite some vanilla spark classes. Therefore, when running a program that uses gluten, it is essential to ensure that +the gluten jar is loaded prior to the vanilla spark jar. In this section, we will provide some configuration settings in +`$SPARK_HOME/conf/spark-defaults.conf` for Yarn client, Yarn cluster, and Local&Standalone mode to guarantee that the gluten jar is prioritized. #### Configurations for Yarn Client mode @@ -63,3 +65,33 @@ spark.driver.extraClassPath={absolute_path}/gluten----SNAPSHOT-jar-with-dependencies.jar ``` + +### Invalid pointer error + +If the below error is reported at runtime, please re-build gluten with `--compile_arrow_java=ON`, then redeploy Gluten jar. + +``` +*** Error in `/usr/local/jdk1.8.0_381/bin/java': free(): invalid pointer: 0x00007f36cb5cec80 *** +======= Backtrace: ========= +/lib64/libc.so.6(+0x7d1fd)[0x7f38c29da1fd] +/lib64/libstdc++.so.6(_ZNSt6locale5_Impl16_M_install_facetEPKNS_2idEPKNS_5facetE+0x142)[0x7f36cb3370d2] +/lib64/libstdc++.so.6(_ZNSt6locale5_ImplC1Em+0x1e3)[0x7f36cb337523] +/lib64/libstdc++.so.6(+0x71495)[0x7f36cb338495] +/lib64/libpthread.so.0(pthread_once+0x50)[0x7f38c3147be0] +/lib64/libstdc++.so.6(+0x714e1)[0x7f36cb3384e1] +/lib64/libstdc++.so.6(_ZNSt6localeC2Ev+0x13)[0x7f36cb338523] +/lib64/libstdc++.so.6(_ZNSt8ios_base4InitC2Ev+0xbc)[0x7f36cb33537c] +/tmp/jnilib-645156599284574767.tmp(+0x2a90)[0x7f375d235a90] +/lib64/ld-linux-x86-64.so.2(+0xf4e3)[0x7f38c33664e3] +/lib64/ld-linux-x86-64.so.2(+0x13b04)[0x7f38c336ab04] +/lib64/ld-linux-x86-64.so.2(+0xf2f4)[0x7f38c33662f4] +/lib64/ld-linux-x86-64.so.2(+0x1321b)[0x7f38c336a21b] +/lib64/libdl.so.2(+0x102b)[0x7f38c2d1f02b] +/lib64/ld-linux-x86-64.so.2(+0xf2f4)[0x7f38c33662f4] +/lib64/libdl.so.2(+0x162d)[0x7f38c2d1f62d] +/lib64/libdl.so.2(dlopen+0x31)[0x7f38c2d1f0c1] +/usr/local/jdk1.8.0_381/jre/lib/amd64/server/libjvm.so(+0x9292b1)[0x7f38c22732b1] +/usr/local/jdk1.8.0_381/jre/lib/amd64/server/libjvm.so(JVM_LoadLibrary+0xa1)[0x7f38c205e0c1] +/usr/local/jdk1.8.0_381/jre/lib/amd64/libjava.so(Java_java_lang_ClassLoader_00024NativeLibrary_load+0x1ac) +... +``` diff --git a/ep/build-velox/src/modify_arrow.patch b/ep/build-velox/src/modify_arrow.patch index e5444d28043f..64d92725d786 100644 --- a/ep/build-velox/src/modify_arrow.patch +++ b/ep/build-velox/src/modify_arrow.patch @@ -31,14 +31,39 @@ index a24f272fe..e25f78c85 100644 #include #include diff --git a/java/pom.xml b/java/pom.xml -index a8328576b..53a70fab8 100644 +index a8328576b..57f282c6c 100644 --- a/java/pom.xml +++ b/java/pom.xml -@@ -1102,6 +1102,7 @@ +@@ -1101,7 +1101,8 @@ + -DARROW_JSON=${ARROW_DATASET} -DARROW_ORC=${ARROW_ORC} -DARROW_PARQUET=${ARROW_PARQUET} - -DARROW_S3=ON +- -DARROW_S3=ON ++ -DARROW_S3=OFF + -DARROW_HDFS=ON -DARROW_SUBSTRAIT=${ARROW_DATASET} -DARROW_USE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release +diff --git a/java/dataset/src/main/cpp/jni_wrapper.cc b/java/dataset/src/main/cpp/jni_wrapper.cc +index d2d976677..eb4b6d1d2 100644 +--- a/java/dataset/src/main/cpp/jni_wrapper.cc ++++ b/java/dataset/src/main/cpp/jni_wrapper.cc +@@ -27,7 +27,9 @@ + #include "arrow/dataset/file_base.h" + #include "arrow/filesystem/localfs.h" + #include "arrow/filesystem/path_util.h" ++#ifdef ARROW_S3 + #include "arrow/filesystem/s3fs.h" ++#endif + #include "arrow/engine/substrait/util.h" + #include "arrow/engine/substrait/serde.h" + #include "arrow/engine/substrait/relation.h" +@@ -622,7 +624,9 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_releaseBuffe + JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_ensureS3Finalized( + JNIEnv* env, jobject) { + JNI_METHOD_START ++#ifdef ARROW_S3 + JniAssertOkOrThrow(arrow::fs::EnsureS3Finalized()); ++#endif + JNI_METHOD_END() + } From d0f611ec6d2f8986de76cc49c7f1c0d85657a438 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 10 May 2024 22:37:55 +0800 Subject: [PATCH 046/402] [GLUTEN-4917][VL][CI] Add TPCDS benchmark (#5693) --- .github/workflows/velox_docker_cache.yml | 2 +- .github/workflows/velox_tpch_bench.yml | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 91676dbb6ade..ec95f48a24f6 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -117,7 +117,7 @@ jobs: - name: CCache after run: | - ccache -vs + ccache -s - uses: actions/cache/save@v3 with: diff --git a/.github/workflows/velox_tpch_bench.yml b/.github/workflows/velox_tpch_bench.yml index 87d71a604fa0..6be12305c623 100644 --- a/.github/workflows/velox_tpch_bench.yml +++ b/.github/workflows/velox_tpch_bench.yml @@ -40,3 +40,18 @@ jobs: run: | branchname=origin/pr/${{ github.event.issue.number }}/head curl --noproxy $JENKINS_HOST -L -X POST http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_Github_Action_Check/build --user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1", "value":"'$branchname'"}]}' + velox-trigger-tpcds-on-comment: + # check the comment if it contains the keywords + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/Benchmark Velox TPCDS') }} + runs-on: velox-self-hosted + container: ubuntu:22.04 + steps: + - uses: actions/checkout@v4 + - run: apt-get update && apt-get install ca-certificates -y && update-ca-certificates + - run: sed -i 's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g' /etc/apt/sources.list + - run: apt-get update + - run: apt-get install -y curl + - name: trigger_tpch_bench + run: | + branchname=origin/pr/${{ github.event.issue.number }}/head + curl --noproxy $JENKINS_HOST -L -X POST http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_DS_Github_Action_Check/build --user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1", "value":"'$branchname'"}]}' From d93100689a152fee685fde4f291de7fc7c400290 Mon Sep 17 00:00:00 2001 From: Tengfei Huang Date: Sat, 11 May 2024 08:05:15 +0800 Subject: [PATCH 047/402] [GLUTEN-4039][VL] Add flatten function support (#5551) --- .../gluten/utils/CHExpressionUtil.scala | 3 ++- .../ScalarFunctionsValidateSuite.scala | 20 +++++++++++++++++++ docs/velox-backend-support-progress.md | 2 +- .../expression/ExpressionMappings.scala | 1 + .../gluten/expression/ExpressionNames.scala | 1 + 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index b4190f1b8d8f..f593e5faceef 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -198,6 +198,7 @@ object CHExpressionUtil { UNIX_MILLIS -> DefaultValidator(), UNIX_MICROS -> DefaultValidator(), TIMESTAMP_MILLIS -> DefaultValidator(), - TIMESTAMP_MICROS -> DefaultValidator() + TIMESTAMP_MICROS -> DefaultValidator(), + FLATTEN -> DefaultValidator() ) } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 233a1ca96b6f..485d70f9de18 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -941,4 +941,24 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("test flatten nested array") { + withTempPath { + path => + Seq[Seq[Seq[Integer]]]( + Seq(Seq(1, 2), Seq(4, 5)), + null, + Seq(null, Seq(1, 2)), + Seq(null, null), + Seq(Seq(1, 2, null), Seq(null, null), Seq(3, 4), Seq.empty)) + .toDF("arrays") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl") + + runQueryAndCompare("select flatten(arrays) as res from array_tbl;") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } } diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 1171b7d91d1a..3d1d25be0ca8 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -281,7 +281,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | explode_outer, explode | | | | | | | | | | | | | | | | | | | | | | | | filter | filter | filter | S | Lambda with index argument not supported | | | | | | | | | | | | | | | | | | | | forall | all_match | | S | | | | | | | | | | | | | | | | | | | | -| flatten | flatten | | | | | | | | | | | | | | | | | | | | | | +| flatten | flatten | flatten | S | | | | | | | | | | | | | | | | | | | | | map | map | map | S | | | | | | | | | | | | | | | | | | | | | map_concat | map_concat | | | | | | | | | | | | | | | | | | | | | | | map_entries | map_entries | | | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index ef43c2724dc5..33e4f0a7b74b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -241,6 +241,7 @@ object ExpressionMappings { Sig[ArrayExists](EXISTS), Sig[Shuffle](SHUFFLE), Sig[ZipWith](ZIP_WITH), + Sig[Flatten](FLATTEN), // Map functions Sig[CreateMap](CREATE_MAP), Sig[GetMapValue](GET_MAP_VALUE), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 54a55b355f36..b9e247a8d439 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -258,6 +258,7 @@ object ExpressionNames { final val TRANSFORM = "transform" final val SHUFFLE = "shuffle" final val ZIP_WITH = "zip_with" + final val FLATTEN = "flatten" // Map functions final val CREATE_MAP = "map" From 34ea806b67c9f0d8692cb105b6f23c25ef202a7f Mon Sep 17 00:00:00 2001 From: Joey Date: Sat, 11 May 2024 09:21:34 +0800 Subject: [PATCH 048/402] [VL] Add InsertIntoHadoopFsRelationCommand test case for csv format (#5681) --- .../apache/gluten/execution/TestOperator.scala | 17 +++++++++++++++++ .../execution/WholeStageTransformerSuite.scala | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 920b8e2bde9f..b69223be1550 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -515,6 +515,23 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + test("insert into select from csv") { + withTable("insert_csv_t") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + spark.sql("create table insert_csv_t(Name string, Language string) using parquet;") + runQueryAndCompare(""" + |insert into insert_csv_t select * from student; + |""".stripMargin) { + checkGlutenOperatorMatch[ArrowFileSourceScanExec] + } + } + } + test("test OneRowRelation") { val df = sql("SELECT 1") checkAnswer(df, Row(1)) diff --git a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index 67c12a6f782a..c52002b68a27 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -22,7 +22,7 @@ import org.apache.gluten.utils.{Arm, FallbackUtil} import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.{DataFrame, GlutenQueryTest, Row} -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.{CommandResultExec, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, ShuffleQueryStageExec} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.DoubleType @@ -222,6 +222,8 @@ abstract class WholeStageTransformerSuite df.queryExecution.executedPlan match { case exec: AdaptiveSparkPlanExec => getChildrenPlan(Seq(exec.executedPlan)) + case cmd: CommandResultExec => + getChildrenPlan(Seq(cmd.commandPhysicalPlan)) case plan => getChildrenPlan(Seq(plan)) } From fafd0ac098030972f6bfb54c3760eeebb92766c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Sat, 11 May 2024 11:17:51 +0800 Subject: [PATCH 049/402] [VL] Rename parsePartitionAndMetadataColumns to parseColumnTypes (#5685) [VL] Rename parsePartitionAndMetadataColumns to parseColumnTypes. --- cpp/velox/substrait/SubstraitParser.cc | 20 +++++++------------ cpp/velox/substrait/SubstraitParser.h | 10 +++++----- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 20 ++++++------------- .../SubstraitToVeloxPlanValidator.cc | 7 +++---- 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 30fe1d7b3a64..5a08d83337ec 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -107,35 +107,29 @@ std::vector SubstraitParser::parseNamedStruct(const ::substrait::NamedS return typeList; } -void SubstraitParser::parsePartitionAndMetadataColumns( +void SubstraitParser::parseColumnTypes( const ::substrait::NamedStruct& namedStruct, - std::vector& isPartitionColumns, - std::vector& isMetadataColumns) { + std::vector& columnTypes) { const auto& columnsTypes = namedStruct.column_types(); if (columnsTypes.size() == 0) { // Regard all columns as regular columns. - isPartitionColumns.resize(namedStruct.names().size(), false); - isMetadataColumns.resize(namedStruct.names().size(), false); + columnTypes.resize(namedStruct.names().size(), ColumnType::kRegular); return; } else { VELOX_CHECK_EQ(columnsTypes.size(), namedStruct.names().size(), "Wrong size for column types and column names."); } - isPartitionColumns.reserve(columnsTypes.size()); - isMetadataColumns.reserve(columnsTypes.size()); + columnTypes.reserve(columnsTypes.size()); for (const auto& columnType : columnsTypes) { switch (columnType) { case ::substrait::NamedStruct::NORMAL_COL: - isPartitionColumns.emplace_back(false); - isMetadataColumns.emplace_back(false); + columnTypes.push_back(ColumnType::kRegular); break; case ::substrait::NamedStruct::PARTITION_COL: - isPartitionColumns.emplace_back(true); - isMetadataColumns.emplace_back(false); + columnTypes.push_back(ColumnType::kPartitionKey); break; case ::substrait::NamedStruct::METADATA_COL: - isPartitionColumns.emplace_back(false); - isMetadataColumns.emplace_back(true); + columnTypes.push_back(ColumnType::kSynthesized); break; default: VELOX_FAIL("Unspecified column type."); diff --git a/cpp/velox/substrait/SubstraitParser.h b/cpp/velox/substrait/SubstraitParser.h index 4aaac5a7159e..1f766b91ca1b 100644 --- a/cpp/velox/substrait/SubstraitParser.h +++ b/cpp/velox/substrait/SubstraitParser.h @@ -28,10 +28,13 @@ #include +#include "velox/connectors/hive/TableHandle.h" #include "velox/type/Type.h" namespace gluten { +typedef ::facebook::velox::connector::hive::HiveColumnHandle::ColumnType ColumnType; + /// This class contains some common functions used to parse Substrait /// components, and convert them into recognizable representations. class SubstraitParser { @@ -41,11 +44,8 @@ class SubstraitParser { const ::substrait::NamedStruct& namedStruct, bool asLowerCase = false); - /// Used to parse partition & metadata columns from Substrait NamedStruct. - static void parsePartitionAndMetadataColumns( - const ::substrait::NamedStruct& namedStruct, - std::vector& isPartitionColumns, - std::vector& isMetadataColumns); + /// Used to parse column types from Substrait NamedStruct. + static void parseColumnTypes(const ::substrait::NamedStruct& namedStruct, std::vector& columnTypes); /// Parse Substrait Type to Velox type. static facebook::velox::TypePtr parseType(const ::substrait::Type& substraitType, bool asLowerCase = false); diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 4b1a2543e7c3..db7c0834dad7 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -595,20 +595,19 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: std::vector tableColumnNames; std::vector partitionedKey; - std::vector isPartitionColumns; - std::vector isMetadataColumns; + std::vector columnTypes; tableColumnNames.reserve(writeRel.table_schema().names_size()); VELOX_CHECK(writeRel.has_table_schema(), "WriteRel should have the table schema to store the column information"); const auto& tableSchema = writeRel.table_schema(); - SubstraitParser::parsePartitionAndMetadataColumns(tableSchema, isPartitionColumns, isMetadataColumns); + SubstraitParser::parseColumnTypes(tableSchema, columnTypes); for (const auto& name : tableSchema.names()) { tableColumnNames.emplace_back(name); } for (int i = 0; i < tableSchema.names_size(); i++) { - if (isPartitionColumns[i]) { + if (columnTypes[i] == ColumnType::kPartitionKey) { partitionedKey.emplace_back(tableColumnNames[i]); } } @@ -1066,8 +1065,7 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: // Get output names and types. std::vector colNameList; std::vector veloxTypeList; - std::vector isPartitionColumns; - std::vector isMetadataColumns; + std::vector columnTypes; // Convert field names into lower case when not case-sensitive. std::shared_ptr veloxCfg = std::make_shared(confMap_); @@ -1083,7 +1081,7 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: colNameList.emplace_back(fieldName); } veloxTypeList = SubstraitParser::parseNamedStruct(baseSchema, asLowerCase); - SubstraitParser::parsePartitionAndMetadataColumns(baseSchema, isPartitionColumns, isMetadataColumns); + SubstraitParser::parseColumnTypes(baseSchema, columnTypes); } // Do not hard-code connector ID and allow for connectors other than Hive. @@ -1138,13 +1136,7 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: std::unordered_map> assignments; for (int idx = 0; idx < colNameList.size(); idx++) { auto outName = SubstraitParser::makeNodeName(planNodeId_, idx); - auto columnType = connector::hive::HiveColumnHandle::ColumnType::kRegular; - if (isPartitionColumns[idx]) { - columnType = connector::hive::HiveColumnHandle::ColumnType::kPartitionKey; - } - if (isMetadataColumns[idx]) { - columnType = connector::hive::HiveColumnHandle::ColumnType::kSynthesized; - } + auto columnType = columnTypes[idx]; assignments[outName] = std::make_shared( colNameList[idx], columnType, veloxTypeList[idx], veloxTypeList[idx]); outNames.emplace_back(outName); diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index ba711e77414e..fc8b912e0c62 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -369,11 +369,10 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::WriteRel& writeR // Validate partition key type. if (writeRel.has_table_schema()) { const auto& tableSchema = writeRel.table_schema(); - std::vector isMetadataColumns; - std::vector isPartitionColumns; - SubstraitParser::parsePartitionAndMetadataColumns(tableSchema, isPartitionColumns, isMetadataColumns); + std::vector columnTypes; + SubstraitParser::parseColumnTypes(tableSchema, columnTypes); for (auto i = 0; i < types.size(); i++) { - if (isPartitionColumns[i]) { + if (columnTypes[i] == ColumnType::kPartitionKey) { switch (types[i]->kind()) { case TypeKind::BOOLEAN: case TypeKind::TINYINT: From 3f95dbffd79d225af596427494ff7a4690935275 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Sat, 11 May 2024 14:50:39 +0800 Subject: [PATCH 050/402] [GLUTEN-5414] [VL] Fix and enable arrow native memory pool track in CSV scan (#5683) --- ep/build-velox/src/modify_arrow.patch | 27 +++++++++++++++++++ .../org/apache/gluten/utils/ArrowUtil.scala | 4 +-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/modify_arrow.patch b/ep/build-velox/src/modify_arrow.patch index 64d92725d786..5814958a936f 100644 --- a/ep/build-velox/src/modify_arrow.patch +++ b/ep/build-velox/src/modify_arrow.patch @@ -30,6 +30,33 @@ index a24f272fe..e25f78c85 100644 #include #include #include +diff --git a/java/dataset/src/main/cpp/jni_wrapper.cc b/java/dataset/src/main/cpp/jni_wrapper.cc +index d2d976677..d7dd01ecd 100644 +--- a/java/dataset/src/main/cpp/jni_wrapper.cc ++++ b/java/dataset/src/main/cpp/jni_wrapper.cc +@@ -126,20 +126,14 @@ class ReserveFromJava : public arrow::dataset::jni::ReservationListener { + : vm_(vm), java_reservation_listener_(java_reservation_listener) {} + + arrow::Status OnReservation(int64_t size) override { +- JNIEnv* env; +- if (vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION) != JNI_OK) { +- return arrow::Status::Invalid("JNIEnv was not attached to current thread"); +- } ++ JNIEnv* env = arrow::dataset::jni::GetEnvOrAttach(vm_); + env->CallObjectMethod(java_reservation_listener_, reserve_memory_method, size); + RETURN_NOT_OK(arrow::dataset::jni::CheckException(env)); + return arrow::Status::OK(); + } + + arrow::Status OnRelease(int64_t size) override { +- JNIEnv* env; +- if (vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION) != JNI_OK) { +- return arrow::Status::Invalid("JNIEnv was not attached to current thread"); +- } ++ JNIEnv* env = arrow::dataset::jni::GetEnvOrAttach(vm_); + env->CallObjectMethod(java_reservation_listener_, unreserve_memory_method, size); + RETURN_NOT_OK(arrow::dataset::jni::CheckException(env)); + return arrow::Status::OK(); diff --git a/java/pom.xml b/java/pom.xml index a8328576b..57f282c6c 100644 --- a/java/pom.xml diff --git a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala index 4579e015b26e..26bebcfae713 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala @@ -18,6 +18,7 @@ package org.apache.gluten.utils import org.apache.gluten.exception.SchemaMismatchException import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.internal.Logging @@ -33,7 +34,6 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.arrow.c.{ArrowSchema, CDataDictionaryProvider, Data} import org.apache.arrow.dataset.file.{FileFormat, FileSystemDatasetFactory} -import org.apache.arrow.dataset.jni.NativeMemoryPool import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.ipc.message.ArrowRecordBatch import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} @@ -144,7 +144,7 @@ object ArrowUtil extends Logging { val allocator = ArrowBufferAllocators.contextInstance() val factory = new FileSystemDatasetFactory( allocator, - NativeMemoryPool.getDefault, // TODO: wait to change + ArrowNativeMemoryPool.arrowPool("FileSystemDatasetFactory"), format, rewriteUri(encodedUri)) factory From ab1f253456df60f3d0bcbb9c966324770c1f62b1 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Sat, 11 May 2024 01:53:11 -0500 Subject: [PATCH 051/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240511) (#5694) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240511) * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/62087 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- .../Functions/SparkFunctionArrayDistinct.cpp | 2 +- cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp | 15 +++++++-------- .../Shuffle/SortedPartitionDataMerger.cpp | 8 ++++---- .../Storages/IO/AggregateSerializationUtils.cpp | 6 +++--- cpp-ch/local-engine/local_engine_jni.cpp | 4 ++-- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index a692c5666aef..aba4b7c567bd 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240509 -CH_COMMIT=81ee5ff107b \ No newline at end of file +CH_BRANCH=rebase_ch/20240511 +CH_COMMIT=725c7a6c78e \ No newline at end of file diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp b/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp index 9ed87f86b3e1..89598ff7a1b1 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp @@ -120,7 +120,7 @@ ColumnPtr FunctionArrayDistinctSpark::executeImpl(const ColumnsWithTypeAndName & IColumn & res_data = res.getData(); ColumnArray::Offsets & res_offsets = res.getOffsets(); - const ColumnNullable * nullable_col = checkAndGetColumn(src_data); + const ColumnNullable * nullable_col = checkAndGetColumn(&src_data); const IColumn * inner_col; diff --git a/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp b/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp index cb5f0111c7e2..6f8df0ecbd75 100644 --- a/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp +++ b/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp @@ -121,9 +121,9 @@ static void writeFixedLengthNullableValue( const std::vector & offsets, const MaskVector & masks = nullptr) { - const auto * nullable_column = checkAndGetColumn(*col.column); - const auto & null_map = nullable_column->getNullMapData(); - const auto & nested_column = nullable_column->getNestedColumn(); + const auto & nullable_column = checkAndGetColumn(*col.column); + const auto & null_map = nullable_column.getNullMapData(); + const auto & nested_column = nullable_column.getNestedColumn(); FixedLengthDataWriter writer(col.type); if (writer.getWhichDataType().isDecimal32()) @@ -215,9 +215,9 @@ static void writeVariableLengthNullableValue( std::vector & buffer_cursor, const MaskVector & masks = nullptr) { - const auto * nullable_column = checkAndGetColumn(*col.column); - const auto & null_map = nullable_column->getNullMapData(); - const auto & nested_column = nullable_column->getNestedColumn(); + const auto & nullable_column = checkAndGetColumn(*col.column); + const auto & null_map = nullable_column.getNullMapData(); + const auto & nested_column = nullable_column.getNestedColumn(); const auto type_without_nullable{removeNullable(col.type)}; const bool use_raw_data = BackingDataLengthCalculator::isDataTypeSupportRawData(type_without_nullable); const bool big_endian = BackingDataLengthCalculator::isBigEndianInSparkRow(type_without_nullable); @@ -331,8 +331,7 @@ SparkRowInfo::SparkRowInfo( if (BackingDataLengthCalculator::isDataTypeSupportRawData(type_without_nullable)) { auto column = col.column->convertToFullIfNeeded(); - const auto * nullable_column = checkAndGetColumn(*column); - if (nullable_column) + if (const auto * nullable_column = checkAndGetColumn(&*column)) { const auto & nested_column = nullable_column->getNestedColumn(); const auto & null_map = nullable_column->getNullMapData(); diff --git a/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.cpp b/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.cpp index 44115cb069d8..9c4ae6bf4680 100644 --- a/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.cpp +++ b/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.cpp @@ -53,16 +53,16 @@ SortedPartitionDataMerger::SortedPartitionDataMerger( int64_t searchLastPartitionIdIndex(ColumnPtr column, size_t start, size_t partition_id) { - const auto * int64_column = checkAndGetColumn(*column); - int64_t low = start, high = int64_column->size() - 1; + const auto & int64_column = checkAndGetColumn(*column); + int64_t low = start, high = int64_column.size() - 1; while (low <= high) { int64_t mid = low + (high - low) / 2; - if (int64_column->get64(mid) > partition_id) + if (int64_column.get64(mid) > partition_id) high = mid - 1; else low = mid + 1; - if (int64_column->get64(high) == partition_id) + if (int64_column.get64(high) == partition_id) return high; } return -1; diff --git a/cpp-ch/local-engine/Storages/IO/AggregateSerializationUtils.cpp b/cpp-ch/local-engine/Storages/IO/AggregateSerializationUtils.cpp index d5e336833536..4a416abe2a1c 100644 --- a/cpp-ch/local-engine/Storages/IO/AggregateSerializationUtils.cpp +++ b/cpp-ch/local-engine/Storages/IO/AggregateSerializationUtils.cpp @@ -51,7 +51,7 @@ bool isFixedSizeAggregateFunction(const DB::AggregateFunctionPtr& function) DB::ColumnWithTypeAndName convertAggregateStateToFixedString(const DB::ColumnWithTypeAndName& col) { - const auto *aggregate_col = checkAndGetColumn(*col.column); + const auto *aggregate_col = checkAndGetColumn(&*col.column); if (!aggregate_col) { return col; @@ -75,7 +75,7 @@ DB::ColumnWithTypeAndName convertAggregateStateToFixedString(const DB::ColumnWit DB::ColumnWithTypeAndName convertAggregateStateToString(const DB::ColumnWithTypeAndName& col) { - const auto *aggregate_col = checkAndGetColumn(*col.column); + const auto *aggregate_col = checkAndGetColumn(&*col.column); if (!aggregate_col) { return col; @@ -130,7 +130,7 @@ DB::Block convertAggregateStateInBlock(DB::Block& block) { if (WhichDataType(item.type).isAggregateFunction()) { - const auto *aggregate_col = checkAndGetColumn(*item.column); + const auto *aggregate_col = checkAndGetColumn(&*item.column); if (isFixedSizeAggregateFunction(aggregate_col->getAggregateFunction())) columns.emplace_back(convertAggregateStateToFixedString(item)); else diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 7baad210e0f3..63341cc53eab 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -398,7 +398,7 @@ Java_org_apache_gluten_vectorized_CHColumnVector_nativeHasNull(JNIEnv * env, job } else { - const auto * nullable = checkAndGetColumn(*col.column); + const auto * nullable = checkAndGetColumn(&*col.column); size_t num_nulls = std::accumulate(nullable->getNullMapData().begin(), nullable->getNullMapData().end(), 0); return num_nulls < block->rows(); } @@ -416,7 +416,7 @@ Java_org_apache_gluten_vectorized_CHColumnVector_nativeNumNulls(JNIEnv * env, jo } else { - const auto * nullable = checkAndGetColumn(*col.column); + const auto * nullable = checkAndGetColumn(&*col.column); return std::accumulate(nullable->getNullMapData().begin(), nullable->getNullMapData().end(), 0); } LOCAL_ENGINE_JNI_METHOD_END(env, -1) From 2efa2e657f2b0cdd43adc9e57963fa17bd7bf5b6 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Sat, 11 May 2024 16:16:48 +0800 Subject: [PATCH 052/402] [GLUTEN-5649][VL] Fix NullPointerException when collect_list / collect_set are partially fallen back (#5655) Fixes #5649. Added vanilla implementation of velox_collect_list and velox_collect_set. Velox backend's collect_list / collect_set implementations require for ARRAY intermediate data however Spark uses BINARY. To address this we did some tricks to forcibly modify the physical plan to change the output schema of partial aggregate operator to align with Velox, but that way the actual information for the two functions in Velox backend is still hidden from query plan so advanced optimizations or compatibility checks are made difficult during planning phase. This patch adds new functions velox_collect_list / velox_collect_set to correctly map to Velox backend's implementation for the two functions and does essential code cleanup and refactors. --- .../clickhouse/CHSparkPlanExecApi.scala | 8 +- .../backendsapi/velox/VeloxBackend.scala | 8 - .../velox/VeloxSparkPlanExecApi.scala | 13 +- .../HashAggregateExecTransformer.scala | 29 ++-- .../expression}/aggregate/HLLAdapter.scala | 5 +- .../expression/aggregate/VeloxCollect.scala | 70 +++++++++ ...omFilterMightContainJointRewriteRule.scala | 3 +- .../gluten/extension/CollectRewriteRule.scala | 106 +++++++++++++ .../FlushableHashAggregateRule.scala | 2 +- .../extension/HLLRewriteRule.scala} | 48 +++--- .../gluten/utils/VeloxIntermediateData.scala | 3 - .../gluten/execution/FallbackSuite.scala | 16 +- .../VeloxAggregateFunctionsSuite.scala | 79 ++++++++++ .../VeloxWindowExpressionSuite.scala | 20 ++- .../substrait/VeloxSubstraitSignature.cc | 1 + .../backendsapi/BackendSettingsApi.scala | 4 - .../expression/ExpressionMappings.scala | 2 - .../columnar/TransformHintRule.scala | 1 + .../columnar/enumerated/RasOffload.scala | 1 + .../columnar/rewrite/RewriteCollect.scala | 140 ------------------ .../{ => columnar/rewrite}/RewriteIn.scala | 4 +- .../columnar/rewrite/RewriteSingleNode.scala | 10 +- .../RewriteTypedImperativeAggregate.scala | 72 --------- .../columnar/validator/FallbackInjects.scala | 38 +++++ .../columnar/validator/Validators.scala | 14 ++ .../gluten/utils/BackendTestSettings.scala | 82 +++++----- .../utils/velox/VeloxTestSettings.scala | 7 +- .../utils/velox/VeloxTestSettings.scala | 7 +- .../utils/velox/VeloxTestSettings.scala | 7 +- .../sql/GlutenDataFrameAggregateSuite.scala | 79 ++++++++-- .../utils/velox/VeloxTestSettings.scala | 7 +- 31 files changed, 522 insertions(+), 364 deletions(-) rename backends-velox/src/main/scala/org/apache/{spark/sql/catalyst/expressions => gluten/expression}/aggregate/HLLAdapter.scala (94%) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxCollect.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/extension/CollectRewriteRule.scala rename backends-velox/src/main/scala/org/apache/{spark/sql/catalyst => gluten/extension}/FlushableHashAggregateRule.scala (99%) rename backends-velox/src/main/scala/org/apache/{spark/sql/catalyst/AggregateFunctionRewriteRule.scala => gluten/extension/HLLRewriteRule.scala} (65%) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala rename gluten-core/src/main/scala/org/apache/gluten/extension/{ => columnar/rewrite}/RewriteIn.scala (96%) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/FallbackInjects.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 030648b06fa6..465041621a61 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -40,7 +40,7 @@ import org.apache.spark.sql.catalyst.{CHAggregateFunctionRewriteRule, EqualToRew import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression +import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, CollectList, CollectSet} import org.apache.spark.sql.catalyst.optimizer.BuildSide import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -631,7 +631,11 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { /** Define backend specfic expression mappings. */ override def extraExpressionMappings: Seq[Sig] = { - SparkShimLoader.getSparkShims.bloomFilterExpressionMappings() + List( + Sig[CollectList](ExpressionNames.COLLECT_LIST), + Sig[CollectSet](ExpressionNames.COLLECT_SET) + ) ++ + SparkShimLoader.getSparkShims.bloomFilterExpressionMappings() } override def genStringTranslateTransformer( diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index aad8ff5d5d55..5509d37e8eb1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -501,14 +501,6 @@ object VeloxBackendSettings extends BackendSettingsApi { override def supportBroadcastNestedLoopJoinExec(): Boolean = true - override def shouldRewriteTypedImperativeAggregate(): Boolean = { - // The intermediate type of collect_list, collect_set in Velox backend is not consistent with - // vanilla Spark, we need to rewrite the aggregate to get the correct data type. - true - } - - override def shouldRewriteCollect(): Boolean = true - override def supportColumnarArrowUdf(): Boolean = true override def generateHdfsConfForLibhdfs(): Boolean = true diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index f98055630304..772f1cfb2422 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -23,8 +23,8 @@ import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.expression.aggregate.VeloxBloomFilterAggregate -import org.apache.gluten.extension.BloomFilterMightContainJointRewriteRule +import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} +import org.apache.gluten.extension.{BloomFilterMightContainJointRewriteRule, CollectRewriteRule, FlushableHashAggregateRule, HLLRewriteRule} import org.apache.gluten.extension.columnar.TransformHints import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, IfThenNode} @@ -37,12 +37,12 @@ import org.apache.spark.serializer.Serializer import org.apache.spark.shuffle.{GenShuffleWriterParameters, GlutenShuffleWriterWrapper} import org.apache.spark.shuffle.utils.ShuffleUtil import org.apache.spark.sql.{SparkSession, Strategy} -import org.apache.spark.sql.catalyst.{AggregateFunctionRewriteRule, FlushableHashAggregateRule, FunctionIdentifier} +import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, HLLAdapter} +import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.optimizer.BuildSide import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -734,7 +734,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { * @return */ override def genExtendedOptimizers(): List[SparkSession => Rule[LogicalPlan]] = List( - AggregateFunctionRewriteRule.apply + CollectRewriteRule.apply, + HLLRewriteRule.apply ) /** @@ -788,6 +789,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { Sig[UDFExpression](ExpressionNames.UDF_PLACEHOLDER), Sig[UserDefinedAggregateFunction](ExpressionNames.UDF_PLACEHOLDER), Sig[NaNvl](ExpressionNames.NANVL), + Sig[VeloxCollectList](ExpressionNames.COLLECT_LIST), + Sig[VeloxCollectSet](ExpressionNames.COLLECT_SET), Sig[VeloxBloomFilterMightContain](ExpressionNames.MIGHT_CONTAIN), Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG) ) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala index f0a7ea1801d1..26d30606ddb7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.extension.columnar.rewrite.RewriteTypedImperativeAggregate +import org.apache.gluten.expression.aggregate.HLLAdapter import org.apache.gluten.substrait.`type`.{TypeBuilder, TypeNode} import org.apache.gluten.substrait.{AggregationParams, SubstraitContext} import org.apache.gluten.substrait.expression.{AggregateFunctionNode, ExpressionBuilder, ExpressionNode, ScalarFunctionNode} @@ -807,25 +807,14 @@ case class HashAggregateExecPullOutHelper( override protected def getAttrForAggregateExprs: List[Attribute] = { aggregateExpressions.zipWithIndex.flatMap { case (expr, index) => - handleSpecialAggregateAttr - .lift(expr) - .getOrElse(expr.mode match { - case Partial | PartialMerge => - expr.aggregateFunction.aggBufferAttributes - case Final => - Seq(aggregateAttributes(index)) - case other => - throw new GlutenNotSupportException(s"Unsupported aggregate mode: $other.") - }) + expr.mode match { + case Partial | PartialMerge => + expr.aggregateFunction.aggBufferAttributes + case Final => + Seq(aggregateAttributes(index)) + case other => + throw new GlutenNotSupportException(s"Unsupported aggregate mode: $other.") + } }.toList } - - private val handleSpecialAggregateAttr: PartialFunction[AggregateExpression, Seq[Attribute]] = { - case ae: AggregateExpression if RewriteTypedImperativeAggregate.shouldRewrite(ae) => - val aggBufferAttr = ae.aggregateFunction.inputAggBufferAttributes.head - Seq( - aggBufferAttr.copy(dataType = ae.aggregateFunction.dataType)( - aggBufferAttr.exprId, - aggBufferAttr.qualifier)) - } } diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HLLAdapter.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/HLLAdapter.scala similarity index 94% rename from backends-velox/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HLLAdapter.scala rename to backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/HLLAdapter.scala index 05e0f84416f0..78b4cb1489a6 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HLLAdapter.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/HLLAdapter.scala @@ -14,10 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.catalyst.expressions.aggregate +package org.apache.gluten.expression.aggregate import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.aggregate.{HyperLogLogPlusPlus, ImperativeAggregate, TypedImperativeAggregate} import org.apache.spark.sql.catalyst.trees.BinaryLike import org.apache.spark.sql.catalyst.util.HyperLogLogPlusPlusHelper import org.apache.spark.sql.types._ @@ -52,7 +53,7 @@ case class HLLAdapter( private lazy val row = new UnsafeRow(hllppHelper.numWords) - override def prettyName: String = "approx_count_distinct_velox" + override def prettyName: String = "velox_approx_count_distinct" override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate = copy(mutableAggBufferOffset = newMutableAggBufferOffset) diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxCollect.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxCollect.scala new file mode 100644 index 000000000000..c12aeab26e70 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxCollect.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.expression.aggregate + +import org.apache.spark.sql.catalyst.expressions.{ArrayDistinct, AttributeReference, Concat, CreateArray, Expression, If, IsNull, Literal} +import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate +import org.apache.spark.sql.catalyst.trees.UnaryLike +import org.apache.spark.sql.types.{ArrayType, DataType} + +abstract class VeloxCollect extends DeclarativeAggregate with UnaryLike[Expression] { + protected lazy val buffer: AttributeReference = AttributeReference("buffer", dataType)() + + override def dataType: DataType = ArrayType(child.dataType, false) + + override def aggBufferAttributes: Seq[AttributeReference] = List(buffer) + + override lazy val initialValues: Seq[Expression] = List(Literal.create(Seq.empty, dataType)) + + override lazy val updateExpressions: Seq[Expression] = List( + If( + IsNull(child), + buffer, + Concat(List(buffer, CreateArray(List(child), useStringTypeWhenEmpty = false)))) + ) + + override lazy val mergeExpressions: Seq[Expression] = List( + Concat(List(buffer.left, buffer.right)) + ) + + override def defaultResult: Option[Literal] = Option(Literal.create(Array(), dataType)) +} + +case class VeloxCollectSet(override val child: Expression) extends VeloxCollect { + override def prettyName: String = "velox_collect_set" + + // Velox's collect_set implementation allows null output. Thus we usually wrap + // the function to enforce non-null output. See CollectRewriteRule#ensureNonNull. + override def nullable: Boolean = true + + override protected def withNewChildInternal(newChild: Expression): Expression = + copy(child = newChild) + + override lazy val evaluateExpression: Expression = + ArrayDistinct(buffer) +} + +case class VeloxCollectList(override val child: Expression) extends VeloxCollect { + override def prettyName: String = "velox_collect_list" + + override def nullable: Boolean = false + + override protected def withNewChildInternal(newChild: Expression): Expression = + copy(child = newChild) + + override val evaluateExpression: Expression = buffer +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala index deba381db78a..9a0a59e8e338 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala @@ -20,13 +20,14 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.expression.VeloxBloomFilterMightContain import org.apache.gluten.expression.aggregate.VeloxBloomFilterAggregate import org.apache.gluten.sql.shims.SparkShimLoader +import org.apache.gluten.utils.PhysicalPlanSelector import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.SparkPlan case class BloomFilterMightContainJointRewriteRule(spark: SparkSession) extends Rule[SparkPlan] { - override def apply(plan: SparkPlan): SparkPlan = { + override def apply(plan: SparkPlan): SparkPlan = PhysicalPlanSelector.maybe(spark, plan) { if (!(GlutenConfig.getConf.enableNativeBloomFilter)) { return plan } diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/CollectRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/CollectRewriteRule.scala new file mode 100644 index 000000000000..d7299c511e15 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/CollectRewriteRule.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension + +import org.apache.gluten.expression.ExpressionMappings +import org.apache.gluten.expression.aggregate.{VeloxCollectList, VeloxCollectSet} +import org.apache.gluten.utils.LogicalPlanSelector + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.{And, Coalesce, Expression, IsNotNull, Literal, WindowExpression} +import org.apache.spark.sql.catalyst.expressions.aggregate._ +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Window} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.types.ArrayType + +import scala.reflect.{classTag, ClassTag} + +/** + * Velox's collect_list / collect_set use array as intermediate data type so aren't compatible with + * vanilla Spark. We here replace the two functions with velox_collect_list / velox_collect_set to + * distinguish. + */ +case class CollectRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { + import CollectRewriteRule._ + override def apply(plan: LogicalPlan): LogicalPlan = LogicalPlanSelector.maybe(spark, plan) { + val out = plan.transformUp { + case node => + val out = replaceCollectSet(replaceCollectList(node)) + out + } + if (out.fastEquals(plan)) { + return plan + } + out + } + + private def replaceCollectList(node: LogicalPlan): LogicalPlan = { + node.transformExpressions { + case func @ AggregateExpression(l: CollectList, _, _, _, _) if has[VeloxCollectList] => + func.copy(VeloxCollectList(l.child)) + } + } + + private def replaceCollectSet(node: LogicalPlan): LogicalPlan = { + // 1. Replace null result from VeloxCollectSet with empty array to align with + // vanilla Spark. + // 2. Filter out null inputs from VeloxCollectSet to align with vanilla Spark. + // + // Since https://github.com/apache/incubator-gluten/pull/4805 + node match { + case agg: Aggregate => + agg.transformExpressions { + case ToVeloxCollectSet(newAggFunc) => + val out = ensureNonNull(newAggFunc) + out + } + case w: Window => + w.transformExpressions { + case func @ WindowExpression(ToVeloxCollectSet(newAggFunc), _) => + val out = ensureNonNull(func.copy(newAggFunc)) + out + } + case other => other + } + } +} + +object CollectRewriteRule { + private def ensureNonNull(expr: Expression): Expression = { + val out = + Coalesce(List(expr, Literal.create(Seq.empty, expr.dataType))) + assert(!out.nullable) + assert(!out.dataType.asInstanceOf[ArrayType].containsNull) + out + } + + private object ToVeloxCollectSet { + def unapply(expr: Expression): Option[Expression] = expr match { + case aggFunc @ AggregateExpression(s: CollectSet, _, _, filter, _) if has[VeloxCollectSet] => + val newFilter = (filter ++ Some(IsNotNull(s.child))).reduceOption(And) + val newAggFunc = + aggFunc.copy(aggregateFunction = VeloxCollectSet(s.child), filter = newFilter) + Some(newAggFunc) + case _ => None + } + } + + private def has[T <: Expression: ClassTag]: Boolean = { + val out = ExpressionMappings.expressionsMap.contains(classTag[T].runtimeClass) + out + } +} diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/catalyst/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala similarity index 99% rename from backends-velox/src/main/scala/org/apache/spark/sql/catalyst/FlushableHashAggregateRule.scala rename to backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index a1858e5999c4..f850b6f457ea 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/catalyst/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.catalyst +package org.apache.gluten.extension import org.apache.gluten.execution.{FlushableHashAggregateExecTransformer, HashAggregateExecTransformer, ProjectExecTransformer, RegularHashAggregateExecTransformer} diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/catalyst/AggregateFunctionRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala similarity index 65% rename from backends-velox/src/main/scala/org/apache/spark/sql/catalyst/AggregateFunctionRewriteRule.scala rename to backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala index c5cb1b24d4cf..cb1e626a1ea6 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/catalyst/AggregateFunctionRewriteRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala @@ -14,37 +14,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.catalyst +package org.apache.gluten.extension import org.apache.gluten.GlutenConfig +import org.apache.gluten.expression.aggregate.HLLAdapter +import org.apache.gluten.utils.LogicalPlanSelector import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Literal -import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, HLLAdapter, HyperLogLogPlusPlus} +import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, HyperLogLogPlusPlus} import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.types._ -case class AggregateFunctionRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { - override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp { - case a: Aggregate => - a.transformExpressions { - case hllExpr @ AggregateExpression(hll: HyperLogLogPlusPlus, _, _, _, _) - if GlutenConfig.getConf.enableNativeHyperLogLogAggregateFunction && - GlutenConfig.getConf.enableColumnarHashAgg && - !hasDistinctAggregateFunc(a) && isDataTypeSupported(hll.child.dataType) => - AggregateExpression( - HLLAdapter( - hll.child, - Literal(hll.relativeSD), - hll.mutableAggBufferOffset, - hll.inputAggBufferOffset), - hllExpr.mode, - hllExpr.isDistinct, - hllExpr.filter, - hllExpr.resultId - ) - } +case class HLLRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { + override def apply(plan: LogicalPlan): LogicalPlan = LogicalPlanSelector.maybe(spark, plan) { + plan.resolveOperatorsUp { + case a: Aggregate => + a.transformExpressions { + case hllExpr @ AggregateExpression(hll: HyperLogLogPlusPlus, _, _, _, _) + if GlutenConfig.getConf.enableNativeHyperLogLogAggregateFunction && + GlutenConfig.getConf.enableColumnarHashAgg && + !hasDistinctAggregateFunc(a) && isDataTypeSupported(hll.child.dataType) => + AggregateExpression( + HLLAdapter( + hll.child, + Literal(hll.relativeSD), + hll.mutableAggBufferOffset, + hll.inputAggBufferOffset), + hllExpr.mode, + hllExpr.isDistinct, + hllExpr.filter, + hllExpr.resultId + ) + } + } } private def hasDistinctAggregateFunc(agg: Aggregate): Boolean = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/VeloxIntermediateData.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/VeloxIntermediateData.scala index a00bcae1ce70..a22655152508 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/VeloxIntermediateData.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/VeloxIntermediateData.scala @@ -125,9 +125,6 @@ object VeloxIntermediateData { aggregateFunc match { case _ @Type(veloxDataTypes: Seq[DataType]) => Seq(StructType(veloxDataTypes.map(StructField("", _)).toArray)) - case _: CollectList | _: CollectSet => - // CollectList and CollectSet should use data type of agg function. - Seq(aggregateFunc.dataType) case _ => // Not use StructType for single column agg intermediate data aggregateFunc.aggBufferAttributes.map(_.dataType) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index fbad525a20dd..e8833a43ca39 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -51,6 +51,12 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl .write .format("parquet") .saveAsTable("tmp2") + spark + .range(100) + .selectExpr("cast(id % 3 as int) as c1", "cast(id % 9 as int) as c2") + .write + .format("parquet") + .saveAsTable("tmp3") } override protected def afterAll(): Unit = { @@ -106,15 +112,14 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl } } - // java.lang.NullPointerException - ignore("fallback final aggregate of collect_list") { + test("fallback final aggregate of collect_list") { withSQLConf( GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1", GlutenConfig.COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR.key -> "false", GlutenConfig.EXPRESSION_BLACK_LIST.key -> "element_at" ) { runQueryAndCompare( - "SELECT sum(ele) FROM (SELECT c1, element_at(collect_list(c2), 1) as ele FROM tmp1 " + + "SELECT sum(ele) FROM (SELECT c1, element_at(collect_list(c2), 1) as ele FROM tmp3 " + "GROUP BY c1)") { df => val columnarToRow = collectColumnarToRow(df.queryExecution.executedPlan) @@ -123,7 +128,8 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl } } - // java.lang.NullPointerException + // Elements in velox_collect_set's output set may be in different order. This is a benign bug + // until we can exactly align with vanilla Spark. ignore("fallback final aggregate of collect_set") { withSQLConf( GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1", @@ -131,7 +137,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl GlutenConfig.EXPRESSION_BLACK_LIST.key -> "element_at" ) { runQueryAndCompare( - "SELECT sum(ele) FROM (SELECT c1, element_at(collect_set(c2), 1) as ele FROM tmp1 " + + "SELECT sum(ele) FROM (SELECT c1, element_at(collect_set(c2), 1) as ele FROM tmp3 " + "GROUP BY c1)") { df => val columnarToRow = collectColumnarToRow(df.queryExecution.executedPlan) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index 70fff52b84d6..398f5e05e0e2 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -17,8 +17,11 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig +import org.apache.gluten.extension.columnar.validator.FallbackInjects import org.apache.spark.SparkConf +import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial} +import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.internal.SQLConf abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSuite { @@ -977,6 +980,82 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu } } + // Used for testing aggregate fallback + sealed trait FallbackMode + case object Offload extends FallbackMode + case object FallbackPartial extends FallbackMode + case object FallbackFinal extends FallbackMode + case object FallbackAll extends FallbackMode + + List(Offload, FallbackPartial, FallbackFinal, FallbackAll).foreach { + mode => + test(s"test fallback collect_set/collect_list with null, $mode") { + mode match { + case Offload => doTest() + case FallbackPartial => + FallbackInjects.fallbackOn { + case agg: BaseAggregateExec => + agg.aggregateExpressions.exists(_.mode == Partial) + } { + doTest() + } + case FallbackFinal => + FallbackInjects.fallbackOn { + case agg: BaseAggregateExec => + agg.aggregateExpressions.exists(_.mode == Final) + } { + doTest() + } + case FallbackAll => + FallbackInjects.fallbackOn { case _: BaseAggregateExec => true } { + doTest() + } + } + + def doTest(): Unit = { + withTempView("collect_tmp") { + Seq((1, null), (1, "a"), (2, null), (3, null), (3, null), (4, "b")) + .toDF("c1", "c2") + .createOrReplaceTempView("collect_tmp") + + // basic test + runQueryAndCompare( + "SELECT collect_set(c2), collect_list(c2) FROM collect_tmp GROUP BY c1") { _ => } + + // test pre project and post project + runQueryAndCompare(""" + |SELECT + |size(collect_set(if(c2 = 'a', 'x', 'y'))) as x, + |size(collect_list(if(c2 = 'a', 'x', 'y'))) as y + |FROM collect_tmp GROUP BY c1 + |""".stripMargin) { _ => } + + // test distinct + runQueryAndCompare( + "SELECT collect_set(c2), collect_list(distinct c2) FROM collect_tmp GROUP BY c1") { + _ => + } + + // test distinct + pre project and post project + runQueryAndCompare(""" + |SELECT + |size(collect_set(if(c2 = 'a', 'x', 'y'))), + |size(collect_list(distinct if(c2 = 'a', 'x', 'y'))) + |FROM collect_tmp GROUP BY c1 + |""".stripMargin) { _ => } + + // test cast array to string + runQueryAndCompare(""" + |SELECT + |cast(collect_set(c2) as string), + |cast(collect_list(c2) as string) + |FROM collect_tmp GROUP BY c1 + |""".stripMargin) { _ => } + } + } + } + } + test("count(1)") { runQueryAndCompare( """ diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxWindowExpressionSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxWindowExpressionSuite.scala index 3dfbd6bd230d..03b295f4983f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxWindowExpressionSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxWindowExpressionSuite.scala @@ -18,6 +18,7 @@ package org.apache.gluten.execution import org.apache.spark.SparkConf import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.types._ class VeloxWindowExpressionSuite extends WholeStageTransformerSuite { @@ -72,7 +73,7 @@ class VeloxWindowExpressionSuite extends WholeStageTransformerSuite { } } - test("collect_list") { + test("collect_list / collect_set") { withTable("t") { val data = Seq( Row(0, 1), @@ -108,6 +109,23 @@ class VeloxWindowExpressionSuite extends WholeStageTransformerSuite { |""".stripMargin) { checkGlutenOperatorMatch[WindowExecTransformer] } + + runQueryAndCompare( + """ + |SELECT + | c1, + | collect_set(c2) OVER ( + | PARTITION BY c1 + | ) + |FROM + | t + |ORDER BY 1, 2; + |""".stripMargin, + noFallBack = false + ) { + // Velox window doesn't support collect_set + checkSparkOperatorMatch[WindowExec] + } } } } diff --git a/cpp/velox/substrait/VeloxSubstraitSignature.cc b/cpp/velox/substrait/VeloxSubstraitSignature.cc index ee7c5f513414..fa415cfef7a8 100644 --- a/cpp/velox/substrait/VeloxSubstraitSignature.cc +++ b/cpp/velox/substrait/VeloxSubstraitSignature.cc @@ -188,6 +188,7 @@ TypePtr VeloxSubstraitSignature::fromSubstraitSignature(const std::string& signa types.emplace_back(fromSubstraitSignature(typeStr)); break; } + VELOX_CHECK(childrenTypes.at(typeEnd) == delimiter) std::string typeStr = childrenTypes.substr(typeStart, typeEnd - typeStart); types.emplace_back(fromSubstraitSignature(typeStr)); typeStart = typeEnd + 1; diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index ac8c2a436f83..c8729561dfe0 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -149,10 +149,6 @@ trait BackendSettingsApi { /** Merge two phases hash based aggregate if need */ def mergeTwoPhasesHashBaseAggregateIfNeed(): Boolean = false - def shouldRewriteTypedImperativeAggregate(): Boolean = false - - def shouldRewriteCollect(): Boolean = false - def supportColumnarArrowUdf(): Boolean = false def generateHdfsConfForLibhdfs(): Boolean = false diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 33e4f0a7b74b..459a3d8e2b41 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -288,8 +288,6 @@ object ExpressionMappings { Sig[MinBy](MIN_BY), Sig[StddevSamp](STDDEV_SAMP), Sig[StddevPop](STDDEV_POP), - Sig[CollectList](COLLECT_LIST), - Sig[CollectSet](COLLECT_SET), Sig[VarianceSamp](VAR_SAMP), Sig[VariancePop](VAR_POP), Sig[BitAndAgg](BIT_AND_AGG), diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index 3c3d23ccc5cc..c9fcc52aa091 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -304,6 +304,7 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { .fallbackComplexExpressions() .fallbackByBackendSettings() .fallbackByUserOptions() + .fallbackByTestInjects() .build() def apply(plan: SparkPlan): SparkPlan = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala index 57e093bdea53..5cabfa88e700 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala @@ -78,6 +78,7 @@ object RasOffload { .fallbackComplexExpressions() .fallbackByBackendSettings() .fallbackByUserOptions() + .fallbackByTestInjects() .build() private val rewrites = RewriteSingleNode.allRules() diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala deleted file mode 100644 index 74d493de5272..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteCollect.scala +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension.columnar.rewrite - -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.utils.PullOutProjectHelper - -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeSet, If, IsNotNull, IsNull, Literal, NamedExpression} -import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.aggregate.BaseAggregateExec -import org.apache.spark.sql.types.ArrayType - -import scala.collection.mutable.ArrayBuffer - -/** - * This rule rewrite collect_set and collect_list to be compatible with vanilla Spark. - * - * - Add `IsNotNull(partial_in)` to skip null value before going to native collect_set - * - Add `If(IsNull(result), CreateArray(Seq.empty), result)` to replace null to empty array - * - * TODO: remove this rule once Velox compatible with vanilla Spark. - */ -object RewriteCollect extends RewriteSingleNode with PullOutProjectHelper { - private lazy val shouldRewriteCollect = - BackendsApiManager.getSettings.shouldRewriteCollect() - - private def shouldAddIsNotNull(ae: AggregateExpression): Boolean = { - ae.aggregateFunction match { - case c: CollectSet if c.child.nullable => - ae.mode match { - case Partial | Complete => true - case _ => false - } - case _ => false - } - } - - private def shouldReplaceNullToEmptyArray(ae: AggregateExpression): Boolean = { - ae.aggregateFunction match { - case _: CollectSet => - ae.mode match { - case Final | Complete => true - case _ => false - } - case _ => false - } - } - - private def shouldRewrite(agg: BaseAggregateExec): Boolean = { - agg.aggregateExpressions.exists { - ae => shouldAddIsNotNull(ae) || shouldReplaceNullToEmptyArray(ae) - } - } - - private def rewriteCollectFilter(aggExprs: Seq[AggregateExpression]): Seq[AggregateExpression] = { - aggExprs - .map { - aggExpr => - if (shouldAddIsNotNull(aggExpr)) { - val newFilter = - (aggExpr.filter ++ Seq(IsNotNull(aggExpr.aggregateFunction.children.head))) - .reduce(And) - aggExpr.copy(filter = Option(newFilter)) - } else { - aggExpr - } - } - } - - private def rewriteAttributesAndResultExpressions( - agg: BaseAggregateExec): (Seq[Attribute], Seq[NamedExpression]) = { - val rewriteAggExprIndices = agg.aggregateExpressions.zipWithIndex - .filter(exprAndIndex => shouldReplaceNullToEmptyArray(exprAndIndex._1)) - .map(_._2) - .toSet - if (rewriteAggExprIndices.isEmpty) { - return (agg.aggregateAttributes, agg.resultExpressions) - } - - assert(agg.aggregateExpressions.size == agg.aggregateAttributes.size) - val rewriteAggAttributes = new ArrayBuffer[Attribute]() - val newAggregateAttributes = agg.aggregateAttributes.zipWithIndex.map { - case (attr, index) => - if (rewriteAggExprIndices.contains(index)) { - rewriteAggAttributes.append(attr) - // We should mark attribute as withNullability since the collect_set and collect_set - // are not nullable but velox may return null. This is to avoid potential issue when - // the post project fallback to vanilla Spark. - attr.withNullability(true) - } else { - attr - } - } - val rewriteAggAttributeSet = AttributeSet(rewriteAggAttributes) - val newResultExpressions = agg.resultExpressions.map { - ne => - val rewritten = ne.transformUp { - case attr: Attribute if rewriteAggAttributeSet.contains(attr) => - assert(attr.dataType.isInstanceOf[ArrayType]) - If(IsNull(attr), Literal.create(Seq.empty, attr.dataType), attr) - } - assert(rewritten.isInstanceOf[NamedExpression]) - rewritten.asInstanceOf[NamedExpression] - } - (newAggregateAttributes, newResultExpressions) - } - - override def rewrite(plan: SparkPlan): SparkPlan = { - if (!shouldRewriteCollect) { - return plan - } - - plan match { - case agg: BaseAggregateExec if shouldRewrite(agg) => - val newAggExprs = rewriteCollectFilter(agg.aggregateExpressions) - val (newAttributes, newResultExprs) = rewriteAttributesAndResultExpressions(agg) - copyBaseAggregateExec(agg)( - newAggregateExpressions = newAggExprs, - newAggregateAttributes = newAttributes, - newResultExpressions = newResultExprs) - - case _ => plan - } - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteIn.scala similarity index 96% rename from gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteIn.scala index 565b9bb19306..da120c39a4a8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/RewriteIn.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteIn.scala @@ -14,9 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.extension - -import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode +package org.apache.gluten.extension.columnar.rewrite import org.apache.spark.sql.catalyst.expressions.{EqualTo, Expression, In, Or} import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, SparkPlan} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala index 73bc8b967fad..01f2e29fe62d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.extension.columnar.rewrite -import org.apache.gluten.extension.RewriteIn - import org.apache.spark.sql.execution.SparkPlan /** @@ -37,12 +35,6 @@ trait RewriteSingleNode { object RewriteSingleNode { def allRules(): Seq[RewriteSingleNode] = { - Seq( - RewriteIn, - RewriteMultiChildrenCount, - RewriteCollect, - RewriteTypedImperativeAggregate, - PullOutPreProject, - PullOutPostProject) + Seq(RewriteIn, RewriteMultiChildrenCount, PullOutPreProject, PullOutPostProject) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala deleted file mode 100644 index 971a87923b23..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteTypedImperativeAggregate.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension.columnar.rewrite - -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.utils.PullOutProjectHelper - -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.aggregate.BaseAggregateExec - -object RewriteTypedImperativeAggregate extends RewriteSingleNode with PullOutProjectHelper { - private lazy val shouldRewriteTypedImperativeAggregate = - BackendsApiManager.getSettings.shouldRewriteTypedImperativeAggregate() - - def shouldRewrite(ae: AggregateExpression): Boolean = { - ae.aggregateFunction match { - case _: CollectList | _: CollectSet => - ae.mode match { - case Partial | PartialMerge => true - case _ => false - } - case _ => false - } - } - - override def rewrite(plan: SparkPlan): SparkPlan = { - if (!shouldRewriteTypedImperativeAggregate) { - return plan - } - - plan match { - case agg: BaseAggregateExec if agg.aggregateExpressions.exists(shouldRewrite) => - val exprMap = agg.aggregateExpressions - .filter(shouldRewrite) - .map(ae => ae.aggregateFunction.inputAggBufferAttributes.head -> ae) - .toMap - val newResultExpressions = agg.resultExpressions.map { - case attr: AttributeReference => - exprMap - .get(attr) - .map { - ae => - attr.copy(dataType = ae.aggregateFunction.dataType)( - exprId = attr.exprId, - qualifier = attr.qualifier - ) - } - .getOrElse(attr) - case other => other - } - copyBaseAggregateExec(agg)(newResultExpressions = newResultExpressions) - - case _ => plan - } - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/FallbackInjects.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/FallbackInjects.scala new file mode 100644 index 000000000000..54139ec9ea1b --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/FallbackInjects.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.validator + +import org.apache.spark.sql.execution.SparkPlan + +object FallbackInjects { + private var fallbackCondition: Option[PartialFunction[SparkPlan, Boolean]] = None + + def fallbackOn[T](condition: PartialFunction[SparkPlan, Boolean])(func: => T): T = + synchronized { + assert(this.fallbackCondition.isEmpty) + this.fallbackCondition = Some(condition) + try { + func + } finally { + this.fallbackCondition = None + } + } + + private[validator] def shouldFallback(node: SparkPlan): Boolean = { + fallbackCondition.exists(_.applyOrElse(node, { _: SparkPlan => false })) + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index 57bcc7e095e0..d4bd9926a84c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -82,6 +82,11 @@ object Validators { this } + def fallbackByTestInjects(): Builder = { + buffer += new FallbackByTestInjects() + this + } + /** Add a custom validator to pipeline. */ def add(validator: Validator): Builder = { buffer += validator @@ -191,6 +196,15 @@ object Validators { } } + private class FallbackByTestInjects() extends Validator { + override def validate(plan: SparkPlan): Validator.OutCome = { + if (FallbackInjects.shouldFallback(plan)) { + return fail(plan) + } + pass() + } + } + private class ValidatorPipeline(validators: Seq[Validator]) extends Validator { override def validate(plan: SparkPlan): Validator.OutCome = { val init: Validator.OutCome = pass() diff --git a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala index fe8d4678dcc8..987635d067be 100644 --- a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala +++ b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/BackendTestSettings.scala @@ -94,14 +94,6 @@ abstract class BackendTestSettings { exclusion.add(Exclude(testNames: _*)) this } - def includeGlutenTest(testName: String*): SuiteSettings = { - inclusion.add(IncludeGlutenTest(testName: _*)) - this - } - def excludeGlutenTest(testName: String*): SuiteSettings = { - exclusion.add(ExcludeGlutenTest(testName: _*)) - this - } def includeByPrefix(prefixes: String*): SuiteSettings = { inclusion.add(IncludeByPrefix(prefixes: _*)) this @@ -110,22 +102,6 @@ abstract class BackendTestSettings { exclusion.add(ExcludeByPrefix(prefixes: _*)) this } - def includeGlutenTestsByPrefix(prefixes: String*): SuiteSettings = { - inclusion.add(IncludeGlutenTestByPrefix(prefixes: _*)) - this - } - def excludeGlutenTestsByPrefix(prefixes: String*): SuiteSettings = { - exclusion.add(ExcludeGlutenTestByPrefix(prefixes: _*)) - this - } - def includeAllGlutenTests(): SuiteSettings = { - inclusion.add(IncludeByPrefix(GLUTEN_TEST)) - this - } - def excludeAllGlutenTests(): SuiteSettings = { - exclusion.add(ExcludeByPrefix(GLUTEN_TEST)) - this - } def disable(reason: String): SuiteSettings = { disableReason = disableReason match { @@ -136,6 +112,40 @@ abstract class BackendTestSettings { } } + object SuiteSettings { + implicit class SuiteSettingsImplicits(settings: SuiteSettings) { + def includeGlutenTest(testName: String*): SuiteSettings = { + settings.include(testName.map(GLUTEN_TEST + _): _*) + settings + } + + def excludeGlutenTest(testName: String*): SuiteSettings = { + settings.exclude(testName.map(GLUTEN_TEST + _): _*) + settings + } + + def includeGlutenTestsByPrefix(prefixes: String*): SuiteSettings = { + settings.includeByPrefix(prefixes.map(GLUTEN_TEST + _): _*) + settings + } + + def excludeGlutenTestsByPrefix(prefixes: String*): SuiteSettings = { + settings.excludeByPrefix(prefixes.map(GLUTEN_TEST + _): _*) + settings + } + + def includeAllGlutenTests(): SuiteSettings = { + settings.include(GLUTEN_TEST) + settings + } + + def excludeAllGlutenTests(): SuiteSettings = { + settings.exclude(GLUTEN_TEST) + settings + } + } + } + protected trait IncludeBase { def isIncluded(testName: String): Boolean } @@ -150,14 +160,6 @@ abstract class BackendTestSettings { val nameSet: Set[String] = Set(testNames: _*) override def isExcluded(testName: String): Boolean = nameSet.contains(testName) } - private case class IncludeGlutenTest(testNames: String*) extends IncludeBase { - val nameSet: Set[String] = testNames.map(name => GLUTEN_TEST + name).toSet - override def isIncluded(testName: String): Boolean = nameSet.contains(testName) - } - private case class ExcludeGlutenTest(testNames: String*) extends ExcludeBase { - val nameSet: Set[String] = testNames.map(name => GLUTEN_TEST + name).toSet - override def isExcluded(testName: String): Boolean = nameSet.contains(testName) - } private case class IncludeByPrefix(prefixes: String*) extends IncludeBase { override def isIncluded(testName: String): Boolean = { if (prefixes.exists(prefix => testName.startsWith(prefix))) { @@ -174,22 +176,6 @@ abstract class BackendTestSettings { false } } - private case class IncludeGlutenTestByPrefix(prefixes: String*) extends IncludeBase { - override def isIncluded(testName: String): Boolean = { - if (prefixes.exists(prefix => testName.startsWith(GLUTEN_TEST + prefix))) { - return true - } - false - } - } - private case class ExcludeGlutenTestByPrefix(prefixes: String*) extends ExcludeBase { - override def isExcluded(testName: String): Boolean = { - if (prefixes.exists(prefix => testName.startsWith(GLUTEN_TEST + prefix))) { - return true - } - false - } - } def getSQLQueryTestSettings: SQLQueryTestSettings } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index a319c5ca9897..1207514c27b2 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -54,7 +54,12 @@ class VeloxTestSettings extends BackendTestSettings { "SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate", // Replaced with another test. "SPARK-19471: AggregationIterator does not initialize the generated result projection" + - " before using it" + " before using it", + // Velox's collect_list / collect_set are by design declarative aggregate so plan check + // for ObjectHashAggregateExec will fail. + "SPARK-22223: ObjectHashAggregate should not introduce unnecessary shuffle", + "SPARK-31620: agg with subquery (whole-stage-codegen = true)", + "SPARK-31620: agg with subquery (whole-stage-codegen = false)" ) enableSuite[GlutenCastSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index d750456b6287..40185aa63476 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -953,7 +953,12 @@ class VeloxTestSettings extends BackendTestSettings { "SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate", // Replaced with another test. "SPARK-19471: AggregationIterator does not initialize the generated result projection" + - " before using it" + " before using it", + // Velox's collect_list / collect_set are by design declarative aggregate so plan check + // for ObjectHashAggregateExec will fail. + "SPARK-22223: ObjectHashAggregate should not introduce unnecessary shuffle", + "SPARK-31620: agg with subquery (whole-stage-codegen = true)", + "SPARK-31620: agg with subquery (whole-stage-codegen = false)" ) enableSuite[GlutenDataFrameAsOfJoinSuite] enableSuite[GlutenDataFrameComplexTypeSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 689eaf39ec7a..47ad21958b6d 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -958,7 +958,12 @@ class VeloxTestSettings extends BackendTestSettings { "SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate", // Replaced with another test. "SPARK-19471: AggregationIterator does not initialize the generated result projection" + - " before using it" + " before using it", + // Velox's collect_list / collect_set are by design declarative aggregate so plan check + // for ObjectHashAggregateExec will fail. + "SPARK-22223: ObjectHashAggregate should not introduce unnecessary shuffle", + "SPARK-31620: agg with subquery (whole-stage-codegen = true)", + "SPARK-31620: agg with subquery (whole-stage-codegen = false)" ) enableSuite[GlutenDataFrameAsOfJoinSuite] enableSuite[GlutenDataFrameComplexTypeSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala index cba70c21f8a3..de56b883442f 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameAggregateSuite.scala @@ -188,21 +188,25 @@ class GlutenDataFrameAggregateSuite extends DataFrameAggregateSuite with GlutenS testGluten("use gluten hash agg to replace vanilla spark sort agg") { withSQLConf(("spark.gluten.sql.columnar.force.hashagg", "false")) { - Seq("A", "B", "C", "D").toDF("col1").createOrReplaceTempView("t1") - // SortAggregateExec is expected to be used for string type input. - val df = spark.sql("select max(col1) from t1") - checkAnswer(df, Row("D") :: Nil) - assert(find(df.queryExecution.executedPlan)(_.isInstanceOf[SortAggregateExec]).isDefined) + withTempView("t1") { + Seq("A", "B", "C", "D").toDF("col1").createOrReplaceTempView("t1") + // SortAggregateExec is expected to be used for string type input. + val df = spark.sql("select max(col1) from t1") + checkAnswer(df, Row("D") :: Nil) + assert(find(df.queryExecution.executedPlan)(_.isInstanceOf[SortAggregateExec]).isDefined) + } } withSQLConf(("spark.gluten.sql.columnar.force.hashagg", "true")) { - Seq("A", "B", "C", "D").toDF("col1").createOrReplaceTempView("t1") - val df = spark.sql("select max(col1) from t1") - checkAnswer(df, Row("D") :: Nil) - // Sort agg is expected to be replaced by gluten's hash agg. - assert( - find(df.queryExecution.executedPlan)( - _.isInstanceOf[HashAggregateExecBaseTransformer]).isDefined) + withTempView("t1") { + Seq("A", "B", "C", "D").toDF("col1").createOrReplaceTempView("t1") + val df = spark.sql("select max(col1) from t1") + checkAnswer(df, Row("D") :: Nil) + // Sort agg is expected to be replaced by gluten's hash agg. + assert( + find(df.queryExecution.executedPlan)( + _.isInstanceOf[HashAggregateExecBaseTransformer]).isDefined) + } } } @@ -279,4 +283,55 @@ class GlutenDataFrameAggregateSuite extends DataFrameAggregateSuite with GlutenS randn(Random.nextLong()) ).foreach(assertNoExceptions) } + + Seq(true, false).foreach { + value => + testGluten(s"SPARK-31620: agg with subquery (whole-stage-codegen = $value)") { + withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> value.toString) { + withTempView("t1", "t2") { + sql("create temporary view t1 as select * from values (1, 2) as t1(a, b)") + sql("create temporary view t2 as select * from values (3, 4) as t2(c, d)") + + // test without grouping keys + checkAnswer( + sql("select sum(if(c > (select a from t1), d, 0)) as csum from t2"), + Row(4) :: Nil) + + // test with grouping keys + checkAnswer( + sql( + "select c, sum(if(c > (select a from t1), d, 0)) as csum from " + + "t2 group by c"), + Row(3, 4) :: Nil) + + // test with distinct + checkAnswer( + sql( + "select avg(distinct(d)), sum(distinct(if(c > (select a from t1)," + + " d, 0))) as csum from t2 group by c"), + Row(4, 4) :: Nil) + + // test subquery with agg + checkAnswer( + sql( + "select sum(distinct(if(c > (select sum(distinct(a)) from t1)," + + " d, 0))) as csum from t2 group by c"), + Row(4) :: Nil) + + // test SortAggregateExec + var df = sql("select max(if(c > (select a from t1), 'str1', 'str2')) as csum from t2") + assert( + find(df.queryExecution.executedPlan)(_.isInstanceOf[SortAggregateExec]).isDefined) + checkAnswer(df, Row("str1") :: Nil) + + // test SortAggregateExec (collect_list) + df = + sql("select collect_list(d), sum(if(c > (select a from t1), d, 0)) as csum from t2") + assert( + find(df.queryExecution.executedPlan)(_.isInstanceOf[SortAggregateExec]).isDefined) + checkAnswer(df, Row(Array(4), 4) :: Nil) + } + } + } + } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index c2385bd56615..2aed0ff78e4b 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -973,7 +973,12 @@ class VeloxTestSettings extends BackendTestSettings { "SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate", // Replaced with another test. "SPARK-19471: AggregationIterator does not initialize the generated result projection" + - " before using it" + " before using it", + // Velox's collect_list / collect_set are by design declarative aggregate so plan check + // for ObjectHashAggregateExec will fail. + "SPARK-22223: ObjectHashAggregate should not introduce unnecessary shuffle", + "SPARK-31620: agg with subquery (whole-stage-codegen = true)", + "SPARK-31620: agg with subquery (whole-stage-codegen = false)" ) enableSuite[GlutenDataFrameAsOfJoinSuite] enableSuite[GlutenDataFrameComplexTypeSuite] From 1c457d5c967f28e9a963d6c72439563b708b66c1 Mon Sep 17 00:00:00 2001 From: WangGuangxin Date: Sat, 11 May 2024 16:51:07 +0800 Subject: [PATCH 053/402] [GLUTEN-5599][VL] Support json_tuple (#5600) --- .../execution/GenerateExecTransformer.scala | 56 +++++++++++++++---- .../gluten/execution/TestOperator.scala | 22 ++++++++ .../expression/ExpressionMappings.scala | 2 + .../expressions/JsonTupleExplode.scala | 24 ++++++++ 4 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/JsonTupleExplode.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala index 830fe396b99a..8f57827423eb 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala @@ -128,7 +128,7 @@ object GenerateExecTransformer { false } else { generator match { - case _: Inline | _: ExplodeBase => + case _: Inline | _: ExplodeBase | _: JsonTuple => true case _ => false @@ -138,9 +138,10 @@ object GenerateExecTransformer { } object PullOutGenerateProjectHelper extends PullOutProjectHelper { + val JSON_PATH_PREFIX = "$." def pullOutPreProject(generate: GenerateExec): SparkPlan = { if (GenerateExecTransformer.supportsGenerate(generate.generator, generate.outer)) { - val newGeneratorChildren = generate.generator match { + generate.generator match { case _: Inline | _: ExplodeBase => val expressionMap = new mutable.HashMap[Expression, NamedExpression]() // The new child should be either the original Attribute, @@ -156,20 +157,51 @@ object PullOutGenerateProjectHelper extends PullOutProjectHelper { // generator.child is other expression, e.g Literal/CreateArray/CreateMap expressionMap.values.head } - Seq(newGeneratorChild) + val newGeneratorChildren = Seq(newGeneratorChild) + + // Avoid using elimainateProjectList to create the project list + // because newGeneratorChild can be a duplicated Attribute in generate.child.output. + // The native side identifies the last field of projection as generator's input. + generate.copy( + generator = + generate.generator.withNewChildren(newGeneratorChildren).asInstanceOf[Generator], + child = ProjectExec(generate.child.output ++ newGeneratorChildren, generate.child) + ) + case JsonTuple(Seq(jsonObj, jsonPaths @ _*)) => + val getJsons: IndexedSeq[Expression] = { + jsonPaths.map { + case jsonPath if jsonPath.foldable => + Option(jsonPath.eval()) match { + case Some(path) => + GetJsonObject(jsonObj, Literal.create(JSON_PATH_PREFIX + path)) + case _ => + Literal.create(null) + } + case jsonPath => + // TODO: The prefix is just for adapting to GetJsonObject. + // Maybe, we can remove this handling in the future by + // making path without "$." recognized + GetJsonObject(jsonObj, Concat(Seq(Literal.create(JSON_PATH_PREFIX), jsonPath))) + }.toIndexedSeq + } + val preGenerateExprs = + Alias( + CreateArray(Seq(CreateStruct(getJsons))), + generatePreAliasName + )() + // use JsonTupleExplode here instead of Explode so that we can distinguish + // JsonTuple and Explode, because JsonTuple has an extra post-projection + val newGenerator = JsonTupleExplode(preGenerateExprs.toAttribute) + generate.copy( + generator = newGenerator, + child = ProjectExec(generate.child.output ++ Seq(preGenerateExprs), generate.child) + ) case _ => // Unreachable. throw new IllegalStateException( s"Generator ${generate.generator.getClass.getSimpleName} is not supported.") } - // Avoid using elimainateProjectList to create the project list - // because newGeneratorChild can be a duplicated Attribute in generate.child.output. - // The native side identifies the last field of projection as generator's input. - generate.copy( - generator = - generate.generator.withNewChildren(newGeneratorChildren).asInstanceOf[Generator], - child = ProjectExec(generate.child.output ++ newGeneratorChildren, generate.child) - ) + } else { generate } @@ -191,7 +223,7 @@ object PullOutGenerateProjectHelper extends PullOutProjectHelper { ProjectExec( (generate.requiredChildOutput :+ ordinal) ++ generate.generatorOutput.tail, newGenerate) - case Inline(_) => + case Inline(_) | JsonTupleExplode(_) => val unnestOutput = { val struct = CreateStruct(generate.generatorOutput) val alias = Alias(struct, generatePostAliasName)() diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index b69223be1550..82d008e1b7ca 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1407,4 +1407,26 @@ class TestOperator extends VeloxWholeStageTransformerSuite { // Verify there is not precision loss for timestamp columns after data broadcast. checkAnswer(df, expected) } + + test("Test json_tuple function") { + withTempView("t") { + Seq[(String)](("{\"a\":\"b\"}"), (null), ("{\"b\":\"a\"}")) + .toDF("json_field") + .createOrReplaceTempView("t") + runQueryAndCompare( + "SELECT * from t lateral view json_tuple(json_field, 'a', 'b') as fa, fb") { + checkGlutenOperatorMatch[GenerateExecTransformer] + } + } + + runQueryAndCompare( + """ + |SELECT + | l_orderkey, + | json_tuple('{"a" : 1, "b" : 2}', CAST(NULL AS STRING), 'b', CAST(NULL AS STRING), 'a') + |from lineitem + |""".stripMargin) { + checkGlutenOperatorMatch[GenerateExecTransformer] + } + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 459a3d8e2b41..1592b0b9aa2d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -214,6 +214,8 @@ object ExpressionMappings { Sig[Sequence](SEQUENCE), Sig[CreateArray](CREATE_ARRAY), Sig[Explode](EXPLODE), + // JsonTupleExplode' behavior are the same with Explode + Sig[JsonTupleExplode](EXPLODE), Sig[Inline](INLINE), Sig[ArrayAggregate](AGGREGATE), Sig[LambdaFunction](LAMBDAFUNCTION), diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/JsonTupleExplode.scala b/gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/JsonTupleExplode.scala new file mode 100644 index 000000000000..b93595cfb5fd --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/JsonTupleExplode.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +/** Used for transforming JsonTuple. The behavior is the same with Explode. */ +case class JsonTupleExplode(child: Expression) extends ExplodeBase { + override val position: Boolean = false + override protected def withNewChildInternal(newChild: Expression): JsonTupleExplode = + copy(child = newChild) +} From 00730ef8fec72111a258cf54d985203cd2820f48 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Sat, 11 May 2024 17:28:04 +0800 Subject: [PATCH 054/402] [VL] Fix async io coredump (#5657) --- .../backendsapi/velox/VeloxListenerApi.scala | 23 ++------------- cpp/velox/benchmarks/GenericBenchmark.cc | 3 ++ cpp/velox/compute/VeloxBackend.h | 7 +++++ cpp/velox/jni/VeloxJniWrapper.cc | 8 ++++++ .../gluten/init/NativeBackendInitializer.java | 28 ++++++++++++++++++- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 1eaf92b5acad..bbeb3a2715fe 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -28,11 +28,9 @@ import org.apache.gluten.vectorized.{JniLibLoader, JniWorkspace} import org.apache.spark.SparkConf import org.apache.spark.sql.execution.datasources.velox.{VeloxOrcWriterInjects, VeloxParquetWriterInjects, VeloxRowSplitter} import org.apache.spark.sql.expression.UDFResolver -import org.apache.spark.sql.internal.GlutenConfigUtil -import org.apache.spark.sql.internal.StaticSQLConf +import org.apache.spark.sql.internal.{GlutenConfigUtil, StaticSQLConf} import org.apache.spark.util.SparkDirectoryUtil -import VeloxListenerApi.initializeNative import org.apache.commons.lang3.StringUtils import scala.sys.process._ @@ -191,7 +189,7 @@ class VeloxListenerApi extends ListenerApi { } val parsed = GlutenConfigUtil.parseConfig(conf.getAll.toMap) - initializeNative(parsed) + NativeBackendInitializer.initializeBackend(parsed) // inject backend-specific implementations to override spark classes // FIXME: The following set instances twice in local mode? @@ -205,19 +203,4 @@ class VeloxListenerApi extends ListenerApi { } } -object VeloxListenerApi { - // Spark DriverPlugin/ExecutorPlugin will only invoke ContextInitializer#initialize method once - // in its init method. - // In cluster mode, ContextInitializer#initialize only will be invoked in different JVM. - // In local mode, ContextInitializer#initialize will be invoked twice in same thread, - // driver first then executor, initFlag ensure only invoke initializeBackend once, - // so there are no race condition here. - private var initFlag: Boolean = false - def initializeNative(conf: Map[String, String]): Unit = { - if (initFlag) { - return - } - NativeBackendInitializer.initializeBackend(conf) - initFlag = true - } -} +object VeloxListenerApi {} diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc index ef88da2c3abf..14593c8dfa12 100644 --- a/cpp/velox/benchmarks/GenericBenchmark.cc +++ b/cpp/velox/benchmarks/GenericBenchmark.cc @@ -26,6 +26,7 @@ #include "benchmarks/common/BenchmarkUtils.h" #include "benchmarks/common/FileReaderIterator.h" +#include "compute/VeloxBackend.h" #include "compute/VeloxPlanConverter.h" #include "compute/VeloxRuntime.h" #include "config/GlutenConfig.h" @@ -260,6 +261,8 @@ auto BM_Generic = [](::benchmark::State& state, writerMetrics.splitTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); state.counters["shuffle_compress_time"] = benchmark::Counter( writerMetrics.compressTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); + + gluten::VeloxBackend::get()->tearDown(); }; int main(int argc, char** argv) { diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h index 288bb5294899..a601d715cfa7 100644 --- a/cpp/velox/compute/VeloxBackend.h +++ b/cpp/velox/compute/VeloxBackend.h @@ -55,6 +55,13 @@ class VeloxBackend { const std::unordered_map& getBackendConf() const; + void tearDown() { + // Destruct IOThreadPoolExecutor will join all threads. + // On threads exit, thread local variables can be constructed with referencing global variables. + // So, we need to destruct IOThreadPoolExecutor and stop the threads before global variables get destructed. + ioExecutor_.reset(); + } + private: explicit VeloxBackend(const std::unordered_map& conf) { init(conf); diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index 188d62ac52d3..7884280c3c94 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -77,6 +77,14 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_init_NativeBackendInitializer_init JNI_METHOD_END() } +JNIEXPORT void JNICALL Java_org_apache_gluten_init_NativeBackendInitializer_shutdown( // NOLINT + JNIEnv* env, + jclass) { + JNI_METHOD_START + gluten::VeloxBackend::get()->tearDown(); + JNI_METHOD_END() +} + JNIEXPORT void JNICALL Java_org_apache_gluten_udf_UdfJniWrapper_getFunctionSignatures( // NOLINT JNIEnv* env, jclass) { diff --git a/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java b/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java index a97af505b632..4863f481f47d 100644 --- a/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java +++ b/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java @@ -19,17 +19,41 @@ import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; +import org.apache.spark.util.GlutenShutdownManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import scala.runtime.BoxedUnit; // Initialize native backend before calling any native methods from Java side. public final class NativeBackendInitializer { - private static final Logger LOG = LoggerFactory.getLogger(NativeBackendInitializer.class); + private static final AtomicBoolean initialized = new AtomicBoolean(false); + // Spark DriverPlugin/ExecutorPlugin will only invoke NativeBackendInitializer#initializeBackend + // method once in its init method. + // In cluster mode, NativeBackendInitializer#initializeBackend only will be invoked in different + // JVM. + // In local mode, NativeBackendInitializer#initializeBackend will be invoked twice in same + // thread, driver first then executor, initialized flag ensure only invoke initializeBackend once, + // so there are no race condition here. public static void initializeBackend(scala.collection.Map conf) { + if (!initialized.compareAndSet(false, true)) { + // Already called. + return; + } + initialize0(conf); + GlutenShutdownManager.addHook( + () -> { + shutdown(); + return BoxedUnit.UNIT; + }); + } + + private static void initialize0(scala.collection.Map conf) { try { String prefix = BackendsApiManager.getSettings().getBackendConfigPrefix(); Map nativeConfMap = GlutenConfig.getNativeBackendConf(prefix, conf); @@ -43,5 +67,7 @@ public static void initializeBackend(scala.collection.Map conf) private static native void initialize(byte[] configPlan); + private static native void shutdown(); + private NativeBackendInitializer() {} } From 4899ea5c8ebc4dd17c1acfa5ab285d3cd5271a70 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Sat, 11 May 2024 20:57:41 +0800 Subject: [PATCH 055/402] [VL] Daily Update Velox Version (2024_05_11) (#5695) Upstream Velox's New Commits: 35c4fa31f by Jimmy Lu, Selective Nimble reader skeleton (9768) 818411304 by zhli1142015, Add shuffle Spark function (9415) 54a060df1 by yan ma, Update aws-sdk-cpp version to 1.11.321(from 1.11.169) (9756) bdbd2555c by Krishna Pai, Fix name of runner used for fuzzer jobs. (9753) f1b6ccf0c by xiaoxmeng, Fix spill read runtime stats unit (9765) --- .github/workflows/velox_docker.yml | 88 ++++++++++++++++++++++++++++++ ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 194cea28727e..a92abda3c4ab 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -497,6 +497,8 @@ jobs: run-spark-test-spark32: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -506,6 +508,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -560,6 +571,8 @@ jobs: run-spark-test-spark32-slow: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -569,6 +582,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -596,6 +618,8 @@ jobs: run-spark-test-spark33: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -605,6 +629,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -651,6 +684,8 @@ jobs: run-spark-test-spark33-slow: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -660,6 +695,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -687,6 +731,8 @@ jobs: run-spark-test-spark34: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -696,6 +742,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -742,6 +797,8 @@ jobs: run-spark-test-spark34-slow: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -751,6 +808,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -778,6 +844,8 @@ jobs: run-spark-test-spark35: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -787,6 +855,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ @@ -832,6 +909,8 @@ jobs: run-spark-test-spark35-slow: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - name: Setup build dependency @@ -841,6 +920,15 @@ jobs: tar -xvf apache-maven-3.8.8-bin.tar.gz mv apache-maven-3.8.8 /usr/lib/maven echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Get Ccache + uses: actions/cache/restore@v3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-centos-release-default + - name: Ensure Cache Dirs Exists + working-directory: ${{ github.workspace }} + run: | + mkdir -p '${{ env.CCACHE_DIR }}' - name: Build Gluten velox third party run: | cd ep/build-velox/src && \ diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index b54da5c68943..c3bacb589fac 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_10 +VELOX_BRANCH=2024_05_11 VELOX_HOME="" #Set on run gluten on HDFS From c44843bb8dac2230045651fdbfd06a57ac9a05e2 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Sun, 12 May 2024 21:26:30 +0800 Subject: [PATCH 056/402] [VL] Daily Update Velox Version (2024_05_12) (#5705) Upstream Velox's New Commits: 2c98308b4 by Kevin Wilfong, Clean up legacy code from Buffer::reallocate and reduce window where Buffer is invalid (9755) ed0ecdd99 by Deepak Majeti, Split Arithmetic Functions Registration (9693) 35c4fa31f by Jimmy Lu, Selective Nimble reader skeleton (9768) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index c3bacb589fac..b5c8fe5dcf35 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_11 +VELOX_BRANCH=2024_05_12 VELOX_HOME="" #Set on run gluten on HDFS From 09950de2dd80090a0bc0fea0631749916614cec1 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Mon, 13 May 2024 09:37:36 +0800 Subject: [PATCH 057/402] [GLUTEN-5414] [VL] Move ArrowFileScanExec class to module backends-velox --- .../velox/VeloxSparkPlanExecApi.scala | 10 ++++-- .../extension/ArrowScanReplaceRule.scala | 34 +++++++++++++++++++ .../execution/ArrowFileSourceScanExec.scala | 0 .../gluten/backendsapi/SparkPlanExecApi.scala | 2 ++ .../columnar/MiscColumnarRules.scala | 4 ++- .../columnar/OffloadSingleNode.scala | 10 +----- .../org/apache/gluten/utils/PlanUtil.scala | 6 +++- 7 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala rename {gluten-core => backends-velox}/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala (100%) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 772f1cfb2422..8d01ab96b845 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -24,7 +24,7 @@ import org.apache.gluten.execution._ import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} -import org.apache.gluten.extension.{BloomFilterMightContainJointRewriteRule, CollectRewriteRule, FlushableHashAggregateRule, HLLRewriteRule} +import org.apache.gluten.extension.{ArrowScanReplaceRule, BloomFilterMightContainJointRewriteRule, CollectRewriteRule, FlushableHashAggregateRule, HLLRewriteRule} import org.apache.gluten.extension.columnar.TransformHints import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, IfThenNode} @@ -744,7 +744,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { * @return */ override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = List( - BloomFilterMightContainJointRewriteRule.apply + BloomFilterMightContainJointRewriteRule.apply, + ArrowScanReplaceRule.apply ) /** @@ -849,4 +850,9 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { case other => other } } + + override def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = plan match { + case _: ArrowFileSourceScanExec => true + case _ => false + } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala new file mode 100644 index 000000000000..2b7c4b1da91b --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension + +import org.apache.gluten.datasource.ArrowCSVFileFormat + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{ArrowFileSourceScanExec, FileSourceScanExec, SparkPlan} + +case class ArrowScanReplaceRule(spark: SparkSession) extends Rule[SparkPlan] { + override def apply(plan: SparkPlan): SparkPlan = { + plan.transformUp { + case plan: FileSourceScanExec if plan.relation.fileFormat.isInstanceOf[ArrowCSVFileFormat] => + ArrowFileSourceScanExec(plan) + case p => p + } + + } +} diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala similarity index 100% rename from gluten-core/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala rename to backends-velox/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index fb2fd961b481..8f2ef19f1408 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -739,4 +739,6 @@ trait SparkPlanExecApi { def genPostProjectForGenerate(generate: GenerateExec): SparkPlan def maybeCollapseTakeOrderedAndProject(plan: SparkPlan): SparkPlan = plan + + def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = false } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala index 068f62e498ce..08c63000ec73 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala @@ -68,7 +68,9 @@ object MiscColumnarRules { case RowToColumnarExec(child) => logDebug(s"ColumnarPostOverrides RowToColumnarExec(${child.getClass})") BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec(child) - case c2r @ ColumnarToRowExec(child) if PlanUtil.outputNativeColumnarData(child) => + case c2r @ ColumnarToRowExec(child) + if PlanUtil.outputNativeColumnarData(child) && + !PlanUtil.outputNativeColumnarSparkCompatibleData(child) => logDebug(s"ColumnarPostOverrides ColumnarToRowExec(${child.getClass})") val nativeC2r = BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(child) if (nativeC2r.doValidate().isValid) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 067aad32cd7a..84a2ec5c6ec8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -297,15 +297,7 @@ object OffloadOthers { class ReplaceSingleNode() extends LogLevelUtil with Logging { def doReplace(p: SparkPlan): SparkPlan = { - val plan = p match { - case plan: FileSourceScanExec - if plan.relation.fileFormat.getClass.getSimpleName == "ArrowCSVFileFormat" => - val arrowScan = ArrowFileSourceScanExec(plan) - TransformHints.tagNotTransformable(arrowScan, "Arrow scan cannot transform") - return arrowScan - case p => p - } - + val plan = p if (TransformHints.isNotTransformable(plan)) { logDebug(s"Columnar Processing for ${plan.getClass} is under row guard.") plan match { diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala index 610f14c86024..4c02687a6fa5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala @@ -16,6 +16,7 @@ */ package org.apache.gluten.utils +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.GlutenPlan import org.apache.spark.sql.execution._ @@ -50,12 +51,15 @@ object PlanUtil { case s: WholeStageCodegenExec => outputNativeColumnarData(s.child) case s: AdaptiveSparkPlanExec => outputNativeColumnarData(s.executedPlan) case i: InMemoryTableScanExec => PlanUtil.isGlutenTableCache(i) - case _: ArrowFileSourceScanExec => false case _: GlutenPlan => true case _ => false } } + def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = { + BackendsApiManager.getSparkPlanExecApiInstance.outputNativeColumnarSparkCompatibleData(plan) + } + def isVanillaColumnarOp(plan: SparkPlan): Boolean = { plan match { case i: InMemoryTableScanExec => From c8c17ddde762bcb34226580c3a0754cdaecd8c19 Mon Sep 17 00:00:00 2001 From: Terry Wang Date: Mon, 13 May 2024 10:43:38 +0800 Subject: [PATCH 058/402] [GLUTEN-5682][VL] Fix incorrect result when isNull & isNotNull coexist in filter (#5670) --- .../scala/org/apache/gluten/execution/TestOperator.scala | 6 ++++++ cpp/velox/substrait/SubstraitToVeloxPlan.cc | 6 +++++- cpp/velox/substrait/SubstraitToVeloxPlan.h | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 82d008e1b7ca..bccb06a130ae 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -106,6 +106,12 @@ class TestOperator extends VeloxWholeStageTransformerSuite { checkLengthAndPlan(df, 6) } + test("is_null and is_not_null coexist") { + val df = runQueryAndCompare( + "select l_orderkey from lineitem where l_comment is null and l_comment is not null") { _ => } + checkLengthAndPlan(df, 0) + } + test("and pushdown") { val df = runQueryAndCompare( "select l_orderkey from lineitem where l_orderkey > 2 " + diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index db7c0834dad7..6ee7ad8278dd 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -2028,6 +2028,7 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( bool nullAllowed = filterInfo.nullAllowed_; bool isNull = filterInfo.isNull_; + bool existIsNullAndIsNotNull = filterInfo.forbidsNullSet_ && filterInfo.isNullSet_; uint32_t rangeSize = std::max(filterInfo.lowerBounds_.size(), filterInfo.upperBounds_.size()); if constexpr (KIND == facebook::velox::TypeKind::HUGEINT) { @@ -2114,7 +2115,10 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( // Handle null filtering. if (rangeSize == 0) { - if (!nullAllowed) { + // handle is not null and is null exists at same time + if (existIsNullAndIsNotNull) { + filters[common::Subfield(inputName)] = std::move(std::make_unique()); + } else if (!nullAllowed) { filters[common::Subfield(inputName)] = std::make_unique(); } else if (isNull) { filters[common::Subfield(inputName)] = std::make_unique(); diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.h b/cpp/velox/substrait/SubstraitToVeloxPlan.h index dc97d2a4cf60..1bda6435eaee 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.h +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.h @@ -316,6 +316,7 @@ class SubstraitToVeloxPlanConverter { if (!initialized_) { initialized_ = true; } + forbidsNullSet_ = true; } // Only null is allowed. @@ -325,6 +326,7 @@ class SubstraitToVeloxPlanConverter { if (!initialized_) { initialized_ = true; } + isNullSet_ = true; } // Return the initialization status. @@ -375,6 +377,8 @@ class SubstraitToVeloxPlanConverter { bool nullAllowed_ = false; bool isNull_ = false; + bool forbidsNullSet_ = false; + bool isNullSet_ = false; // If true, left bound will be exclusive. std::vector lowerExclusives_; From 7324ffe221ee8aa2ba0985011e1a4146d0c9995a Mon Sep 17 00:00:00 2001 From: zhouyifan279 <88070094+zhouyifan279@users.noreply.github.com> Date: Mon, 13 May 2024 12:54:32 +0800 Subject: [PATCH 059/402] [BUILD] Remove duplicated arrow-dataset dependency from gluten-data/pom.xml (#5703) --- gluten-data/pom.xml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml index 1e4438b84b4f..bb84a06b4125 100644 --- a/gluten-data/pom.xml +++ b/gluten-data/pom.xml @@ -165,34 +165,6 @@ compile - - org.apache.arrow - arrow-dataset - ${arrow.version} - - - io.netty - netty-common - - - io.netty - netty-buffer - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - protobuf-java - com.google.protobuf - - - compile - org.apache.hadoop hadoop-common From 74f3061d88ce3155daca921c23266801533e7710 Mon Sep 17 00:00:00 2001 From: James Xu Date: Mon, 13 May 2024 13:38:28 +0800 Subject: [PATCH 060/402] [GLUTEN-5708][VL] Minor wording polishing for NewToGluten.md (#5707) --- docs/developers/NewToGluten.md | 62 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 1eb21d1e6c05..a397003adf36 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -6,22 +6,20 @@ parent: Developer Overview --- Help users to debug and test with gluten. -For intel internal developer, you could refer to internal wiki [New Employee Guide](https://wiki.ith.intel.com/display/HPDA/New+Employee+Guide) to get more information such as proxy settings, -Gluten has cpp code and java/scala code, we can use some useful IDE to read and debug. - # Environment Now gluten supports Ubuntu20.04, Ubuntu22.04, centos8, centos7 and macOS. -## Openjdk8 +## OpenJDK 8 -### Environment setting +### Environment Setting -For root user, the environment variables file is `/etc/profile`, it will make effect for all the users. +For root user, the environment variables file is `/etc/profile`, it will take effect for all the users. For other user, you can set in `~/.bashrc`. -### Guide for ubuntu +### Guide for Ubuntu + The default JDK version in ubuntu is java11, we need to set to java8. ```bash @@ -43,9 +41,9 @@ export PATH="$PATH:$JAVA_HOME/bin" > Must set PATH with double quote in ubuntu. -## Openjdk17 +## OpenJDK 17 -By defaults, Gluten compiles package using JDK8. Add maven profile `-Pjava-17` changing to use JDK17, and please make sure your JAVA_HOME points to jdk17. +By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17, and please make sure your JAVA_HOME points to jdk17. Apache Spark and Arrow requires setting java args `-Dio.netty.tryReflectionSetAccessible=true`, see [SPARK-29924](https://issues.apache.org/jira/browse/SPARK-29924) and [ARROW-6206](https://issues.apache.org/jira/browse/ARROW-6206). So please add following configs in `spark-defaults.conf`: @@ -78,31 +76,20 @@ If you need to debug the tests in /gluten-ut, You need to compile java c # Java/scala code development with Intellij -## Linux intellij local debug +## Linux IntelliJ local debug -Install the linux intellij version, and debug code locally. +Install the Linux IntelliJ version, and debug code locally. - Ask your linux maintainer to install the desktop, and then restart the server. - If you use Moba-XTerm to connect linux server, you don't need to install x11 server, If not (e.g. putty), please follow this guide: [X11 Forwarding: Setup Instructions for Linux and Mac](https://www.businessnewsdaily.com/11035-how-to-use-x11-forwarding.html) -- Download [intellij linux community version](https://www.jetbrains.com/idea/download/?fromIDE=#section=linux) to linux server +- Download [IntelliJ Linux community version](https://www.jetbrains.com/idea/download/?fromIDE=#section=linux) to Linux server - Start Idea, `bash /idea.sh` -Notes: Sometimes, your desktop may stop accidently, left idea running. - -```bash -root@xx2:~bash idea-IC-221.5787.30/bin/idea.sh -Already running -root@xx2:~ps ux | grep intellij -root@xx2:kill -9 -``` - -And then restart idea. +## Windows/macOS IntelliJ remote debug -## Windows/Mac intellij remote debug - -If you have Ultimate intellij, you can try to debug remotely. +If you have IntelliJ Ultimate Edition, you can debug Gluten code remotely. ## Set up gluten project @@ -113,8 +100,8 @@ If you have Ultimate intellij, you can try to debug remotely. ## Java/Scala code style -Intellij IDE supports importing settings for Java/Scala code style. You can import [intellij-codestyle.xml](../../dev/intellij-codestyle.xml) to your IDE. -See [Intellij guide](https://www.jetbrains.com/help/idea/configuring-code-style.html#import-code-style). +IntelliJ supports importing settings for Java/Scala code style. You can import [intellij-codestyle.xml](../../dev/intellij-codestyle.xml) to your IDE. +See [IntelliJ guide](https://www.jetbrains.com/help/idea/configuring-code-style.html#import-code-style). To generate a fix for Java/Scala code style, you can run one or more of the below commands according to the code modules involved in your PR. @@ -161,7 +148,7 @@ VSCode support 2 ways to set user setting. ### Build by vscode -VSCode will try to compile the debug version in /build. +VSCode will try to compile using debug mode in /build. And we need to compile velox debug mode before, if you have compiled velox release mode, you just need to do. ```bash @@ -259,14 +246,15 @@ Then you can create breakpoint and debug in `Run and Debug` section. ### Velox debug -For some velox tests such as `ParquetReaderTest`, tests need to read the parquet file in `/velox/dwio/parquet/tests/examples`, you should let the screen on `ParquetReaderTest.cpp`, then click `Start Debuging`, otherwise you will raise No such file or directory exception +For some velox tests such as `ParquetReaderTest`, tests need to read the parquet file in `/velox/dwio/parquet/tests/examples`, +you should let the screen on `ParquetReaderTest.cpp`, then click `Start Debuging`, otherwise `No such file or directory` exception will be raised. -## Usefule notes +## Useful notes -### Upgrade vscode +### Do not upgrade vscode No need to upgrade vscode version, if upgraded, will download linux server again, switch update mode to off -Search `update` in Manage->Settings to turn off update mode +Search `update` in Manage->Settings to turn off update mode. ### Colour setting @@ -299,7 +287,7 @@ Set config in `settings.json` If exists multiple clang-format version, formatOnSave may not take effect, specify the default formatter Search `default formatter` in `Settings`, select Clang-Format. -If your formatOnSave still make no effect, you can use shortcut `SHIFT+ALT+F` to format one file mannually. +If your formatOnSave still make no effect, you can use shortcut `SHIFT+ALT+F` to format one file manually. # Debug cpp code with coredump @@ -370,7 +358,9 @@ wait to attach.... ``` # Debug Memory leak + ## Arrow memory allocator leak + If you receive error message like ```bash @@ -378,6 +368,7 @@ If you receive error message like 24/04/18 08:15:38 WARN ArrowBufferAllocators$ArrowBufferAllocatorManager: Leaked allocator stack Allocator(ROOT) 0/191/319/9223372036854775807 (res/actual/peak/limit) ``` You can open the Arrow allocator debug config by add VP option `-Darrow.memory.debug.allocator=true`, then you can get more details like + ```bash child allocators: 0 ledgers: 7 @@ -403,9 +394,12 @@ child allocators: 0 at org.apache.gluten.utils.IteratorCompleter.hasNext(Iterators.scala:69) at org.apache.spark.memory.SparkMemoryUtil$UnsafeItr.hasNext(SparkMemoryUtil.scala:246) ``` + ## CPP code memory leak + Sometimes you cannot get the coredump symbols, if you debug memory leak, you can write googletest to use valgrind to detect -``` + +```bash apt install valgrind valgrind --leak-check=yes ./exec_backend_test ``` From f5a6c98bb80b11f22a8e07abf9ee22852fc5a425 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 13 May 2024 15:33:21 +0800 Subject: [PATCH 061/402] [CORE] Add a compilation-time check to forbid case-class inheritance --- .../spark/shuffle/HashPartitioningWrapper.scala | 1 + .../sql/delta/catalog/ClickHouseTableV2.scala | 2 ++ .../v2/clickhouse/metadata/AddFileTags.scala | 1 + pom.xml | 15 +++++++++++++++ 4 files changed, 19 insertions(+) rename {gluten-core => backends-clickhouse}/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala (94%) diff --git a/gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala similarity index 94% rename from gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala index 06ce8fe0f508..bf1cbe4a8046 100644 --- a/gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning // A wrapper for HashPartitioning to remain original hash expressions. // Only used by CH backend when shuffle hash expressions contains non-field expression. +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class HashPartitioningWrapper( original: Seq[Expression], newExpr: Seq[Expression], diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 8c09ae7c7817..1107c6a2ef76 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -39,6 +39,7 @@ import java.{util => ju} import scala.collection.JavaConverters._ +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class ClickHouseTableV2( override val spark: SparkSession, override val path: Path, @@ -268,6 +269,7 @@ class ClickHouseTableV2( } } +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class TempClickHouseTableV2( override val spark: SparkSession, override val catalogTable: Option[CatalogTable] = None) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala index bdb3a30e914b..0680663eb553 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala @@ -28,6 +28,7 @@ import java.util.{List => JList} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class AddMergeTreeParts( val database: String, val table: String, diff --git a/pom.xml b/pom.xml index dbf46ac176f8..a29f19832e35 100644 --- a/pom.xml +++ b/pom.xml @@ -615,6 +615,20 @@ scala-maven-plugin ${scala.compiler.version} + + + org.wartremover + wartremover_${scala.binary.version} + 3.1.6 + + + + + io.github.zhztheplayer.scalawarts + scalawarts + 0.1.0 + + ${scala.recompile.mode} -Wconf:msg=While parsing annotations in:silent @@ -622,6 +636,7 @@ -Xfatal-warnings -deprecation -feature + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass From 6db8920e6bcb423724fbbb9888ad71f663ba3053 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Mon, 13 May 2024 15:41:23 +0800 Subject: [PATCH 062/402] [VL] Add test for shuffle function (#5722) [VL] Add test for shuffle function. --- .../velox/VeloxSparkPlanExecApi.scala | 10 ++++++++++ .../execution/ScalarFunctionsValidateSuite.scala | 16 ++++++++++++++++ docs/velox-backend-support-progress.md | 2 +- .../gluten/backendsapi/SparkPlanExecApi.scala | 7 +++++++ .../gluten/expression/ExpressionConverter.scala | 6 ++++++ .../gluten/utils/velox/VeloxTestSettings.scala | 3 --- .../gluten/utils/velox/VeloxTestSettings.scala | 3 --- .../gluten/utils/velox/VeloxTestSettings.scala | 3 --- .../gluten/utils/velox/VeloxTestSettings.scala | 3 --- 9 files changed, 40 insertions(+), 13 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 8d01ab96b845..4d41ed0c0a79 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -143,6 +143,16 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { original) } + override def genShuffleTransformer( + substraitExprName: String, + child: ExpressionTransformer, + original: Shuffle): ExpressionTransformer = { + GenericExpressionTransformer( + substraitExprName, + Seq(child, LiteralTransformer(Literal(original.randomSeed.get))), + original) + } + override def genTryAddTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 485d70f9de18..834e172f86bd 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -878,6 +878,22 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("test shuffle") { + withTempPath { + path => + Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null) + .toDF("value") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl") + + runQueryAndCompare("select shuffle(value) from array_tbl;", false) { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("negative") { runQueryAndCompare("select negative(l_orderkey) from lineitem") { checkGlutenOperatorMatch[ProjectExecTransformer] diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 3d1d25be0ca8..8b640c081da5 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -295,7 +295,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | named_struct,struct | row_construct | named_struct | S | | | | | | | | | | | | | | | | | | S | | | posexplode_outer,posexplode | | | | | | | | | | | | | | | | | | | | | | | | sequence | | | | | | | | | | | | | | | | | | | | | | | -| shuffle | shuffle | | | | | | | | | | | | | | | | | | | | | | +| shuffle | shuffle | shuffle | S | | | | | | | | | | | | | | | | | | | | | size | | size | S | | | | | | | | | | | | | | | | | | | | | slice | slice | | | | | | | | | | | | | | | | | | | | | | | sort_array | | sort_array | S | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 8f2ef19f1408..c2c733070688 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -211,6 +211,13 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(), original) } + def genShuffleTransformer( + substraitExprName: String, + child: ExpressionTransformer, + original: Shuffle): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(child), original) + } + def genTryAddTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 562ae294e2c0..495fbf8d5a4b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -652,6 +652,12 @@ object ExpressionConverter extends SQLConfHelper with Logging { a ) + case s: Shuffle => + BackendsApiManager.getSparkPlanExecApiInstance.genShuffleTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(s.child, attributeSeq, expressionsMap), + s + ) case expr => GenericExpressionTransformer( substraitExprName, diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 1207514c27b2..dbd7dc187ba5 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -280,9 +280,6 @@ class VeloxTestSettings extends BackendTestSettings { // blocked by Velox-5768 .exclude("aggregate function - array for primitive type containing null") .exclude("aggregate function - array for non-primitive type") - .exclude("shuffle function - array for primitive type not containing null") - .exclude("shuffle function - array for primitive type containing null") - .exclude("shuffle function - array for non-primitive type") // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameTungstenSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 40185aa63476..9b469a98d137 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -966,9 +966,6 @@ class VeloxTestSettings extends BackendTestSettings { // blocked by Velox-5768 .exclude("aggregate function - array for primitive type containing null") .exclude("aggregate function - array for non-primitive type") - .exclude("shuffle function - array for primitive type not containing null") - .exclude("shuffle function - array for primitive type containing null") - .exclude("shuffle function - array for non-primitive type") // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameHintSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 47ad21958b6d..498ed5ef4da4 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -971,9 +971,6 @@ class VeloxTestSettings extends BackendTestSettings { // blocked by Velox-5768 .exclude("aggregate function - array for primitive type containing null") .exclude("aggregate function - array for non-primitive type") - .exclude("shuffle function - array for primitive type not containing null") - .exclude("shuffle function - array for primitive type containing null") - .exclude("shuffle function - array for non-primitive type") // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameHintSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 2aed0ff78e4b..a5981941146e 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -986,9 +986,6 @@ class VeloxTestSettings extends BackendTestSettings { // blocked by Velox-5768 .exclude("aggregate function - array for primitive type containing null") .exclude("aggregate function - array for non-primitive type") - .exclude("shuffle function - array for primitive type not containing null") - .exclude("shuffle function - array for primitive type containing null") - .exclude("shuffle function - array for non-primitive type") // Rewrite this test because Velox sorts rows by key for primitive data types, which disrupts the original row sequence. .exclude("map_zip_with function - map of primitive types") enableSuite[GlutenDataFrameHintSuite] From 512f4e1508d682ffa1aa64daf62551d86e06732c Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 13 May 2024 15:50:04 +0800 Subject: [PATCH 063/402] Revert "[CORE] Add a compilation-time check to forbid case-class inheritance" (#5727) This reverts commit f5a6c98bb80b11f22a8e07abf9ee22852fc5a425. --- .../sql/delta/catalog/ClickHouseTableV2.scala | 2 -- .../v2/clickhouse/metadata/AddFileTags.scala | 1 - .../spark/shuffle/HashPartitioningWrapper.scala | 1 - pom.xml | 15 --------------- 4 files changed, 19 deletions(-) rename {backends-clickhouse => gluten-core}/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala (94%) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 1107c6a2ef76..8c09ae7c7817 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -39,7 +39,6 @@ import java.{util => ju} import scala.collection.JavaConverters._ -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class ClickHouseTableV2( override val spark: SparkSession, override val path: Path, @@ -269,7 +268,6 @@ class ClickHouseTableV2( } } -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class TempClickHouseTableV2( override val spark: SparkSession, override val catalogTable: Option[CatalogTable] = None) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala index 0680663eb553..bdb3a30e914b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala @@ -28,7 +28,6 @@ import java.util.{List => JList} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class AddMergeTreeParts( val database: String, val table: String, diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala b/gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala similarity index 94% rename from backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala rename to gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala index bf1cbe4a8046..06ce8fe0f508 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala +++ b/gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala @@ -21,7 +21,6 @@ import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning // A wrapper for HashPartitioning to remain original hash expressions. // Only used by CH backend when shuffle hash expressions contains non-field expression. -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class HashPartitioningWrapper( original: Seq[Expression], newExpr: Seq[Expression], diff --git a/pom.xml b/pom.xml index a29f19832e35..dbf46ac176f8 100644 --- a/pom.xml +++ b/pom.xml @@ -615,20 +615,6 @@ scala-maven-plugin ${scala.compiler.version} - - - org.wartremover - wartremover_${scala.binary.version} - 3.1.6 - - - - - io.github.zhztheplayer.scalawarts - scalawarts - 0.1.0 - - ${scala.recompile.mode} -Wconf:msg=While parsing annotations in:silent @@ -636,7 +622,6 @@ -Xfatal-warnings -deprecation -feature - -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass From 600a5eeb93cf2cbd12aa2c018d28addf12510bd2 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 13 May 2024 18:19:46 +0800 Subject: [PATCH 064/402] [VL]: Fix VeloxColumnarWriteFilesExecwithNewChildren doesn't replace the dummy child (#5726) --- .../VeloxColumnarWriteFilesExec.scala | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala index 23dff990c464..1d3d55afb526 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala @@ -259,8 +259,9 @@ class VeloxColumnarWriteFilesRDD( // we need to expose a dummy child (as right child) with type "WriteFilesExec" to let Spark // choose the new write code path (version >= 3.4). The actual plan to write is the left child // of this operator. -case class VeloxColumnarWriteFilesExec( - child: SparkPlan, +case class VeloxColumnarWriteFilesExec private ( + override val left: SparkPlan, + override val right: SparkPlan, fileFormat: FileFormat, partitionColumns: Seq[Attribute], bucketSpec: Option[BucketSpec], @@ -269,7 +270,8 @@ case class VeloxColumnarWriteFilesExec( extends BinaryExecNode with GlutenPlan with VeloxColumnarWriteFilesExec.ExecuteWriteCompatible { - import VeloxColumnarWriteFilesExec._ + + val child: SparkPlan = left override lazy val references: AttributeSet = AttributeSet.empty @@ -320,28 +322,49 @@ case class VeloxColumnarWriteFilesExec( new VeloxColumnarWriteFilesRDD(rdd, writeFilesSpec, jobTrackerID) } } - - override def left: SparkPlan = child - - // This is a workaround for FileFormatWriter#write. Vanilla Spark (version >= 3.4) requires for - // a plan that has at least one node exactly of type `WriteFilesExec` that is a Scala case-class, - // to decide to choose new `#executeWrite` code path over the legacy `#execute` for write - // operation. - // - // So we add a no-op `WriteFilesExec` child to let Spark pick the new code path. - // - // See: FileFormatWriter#write - // See: V1Writes#getWriteFilesOpt - override val right: SparkPlan = - WriteFilesExec(NoopLeaf(), fileFormat, partitionColumns, bucketSpec, options, staticPartitions) - override protected def withNewChildrenInternal( newLeft: SparkPlan, newRight: SparkPlan): SparkPlan = - copy(newLeft, fileFormat, partitionColumns, bucketSpec, options, staticPartitions) + copy(newLeft, newRight, fileFormat, partitionColumns, bucketSpec, options, staticPartitions) } object VeloxColumnarWriteFilesExec { + + def apply( + child: SparkPlan, + fileFormat: FileFormat, + partitionColumns: Seq[Attribute], + bucketSpec: Option[BucketSpec], + options: Map[String, String], + staticPartitions: TablePartitionSpec): VeloxColumnarWriteFilesExec = { + // This is a workaround for FileFormatWriter#write. Vanilla Spark (version >= 3.4) requires for + // a plan that has at least one node exactly of type `WriteFilesExec` that is a Scala + // case-class, to decide to choose new `#executeWrite` code path over the legacy `#execute` + // for write operation. + // + // So we add a no-op `WriteFilesExec` child to let Spark pick the new code path. + // + // See: FileFormatWriter#write + // See: V1Writes#getWriteFilesOpt + val right: SparkPlan = + WriteFilesExec( + NoopLeaf(), + fileFormat, + partitionColumns, + bucketSpec, + options, + staticPartitions) + + VeloxColumnarWriteFilesExec( + child, + right, + fileFormat, + partitionColumns, + bucketSpec, + options, + staticPartitions) + } + private case class NoopLeaf() extends LeafExecNode { override protected def doExecute(): RDD[InternalRow] = throw new GlutenException(s"$nodeName does not support doExecute") From 33f993554bebc388c7011dd91b86eaadc729f0d5 Mon Sep 17 00:00:00 2001 From: zhouyifan279 <88070094+zhouyifan279@users.noreply.github.com> Date: Mon, 13 May 2024 19:03:41 +0800 Subject: [PATCH 065/402] [GLUTEN-4652][VL] Fix min_by/max_by result mismatch when RDD partition num > 1 (#5711) --- .../VeloxAggregateFunctionsSuite.scala | 18 +++------ .../functions/RegistrationAllFunctions.cc | 23 +++++++----- .../functions/RowConstructorWithAllNull.h | 37 ------------------- .../functions/RowConstructorWithNull.cc | 10 +---- .../functions/RowConstructorWithNull.h | 8 ++++ 5 files changed, 28 insertions(+), 68 deletions(-) delete mode 100644 cpp/velox/operators/functions/RowConstructorWithAllNull.h diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index 398f5e05e0e2..faa361edf5aa 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -194,18 +194,12 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu } test("min_by/max_by") { - withTempPath { - path => - Seq((5: Integer, 6: Integer), (null: Integer, 11: Integer), (null: Integer, 5: Integer)) - .toDF("a", "b") - .write - .parquet(path.getCanonicalPath) - spark.read - .parquet(path.getCanonicalPath) - .createOrReplaceTempView("test") - runQueryAndCompare("select min_by(a, b), max_by(a, b) from test") { - checkGlutenOperatorMatch[HashAggregateExecTransformer] - } + withSQLConf(("spark.sql.leafNodeDefaultParallelism", "2")) { + runQueryAndCompare( + "select min_by(a, b), max_by(a, b) from " + + "values (5, 6), (null, 11), (null, 5) test(a, b)") { + checkGlutenOperatorMatch[HashAggregateExecTransformer] + } } } diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index c77fa47e5bff..5a6b0f6aa2e7 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -15,11 +15,10 @@ * limitations under the License. */ #include "operators/functions/RegistrationAllFunctions.h" + #include "operators/functions/Arithmetic.h" -#include "operators/functions/RowConstructorWithAllNull.h" #include "operators/functions/RowConstructorWithNull.h" #include "operators/functions/RowFunctionWithNull.h" - #include "velox/expression/SpecialFormRegistry.h" #include "velox/expression/VectorFunction.h" #include "velox/functions/lib/RegistrationHelpers.h" @@ -45,29 +44,32 @@ void registerFunctionOverwrite() { velox::registerFunction({"round"}); velox::registerFunction({"round"}); + auto kRowConstructorWithNull = RowConstructorWithNullCallToSpecialForm::kRowConstructorWithNull; velox::exec::registerVectorFunction( - "row_constructor_with_null", + kRowConstructorWithNull, std::vector>{}, std::make_unique>(), RowFunctionWithNull::metadata()); velox::exec::registerFunctionCallToSpecialForm( - RowConstructorWithNullCallToSpecialForm::kRowConstructorWithNull, - std::make_unique()); + kRowConstructorWithNull, std::make_unique(kRowConstructorWithNull)); + + auto kRowConstructorWithAllNull = RowConstructorWithNullCallToSpecialForm::kRowConstructorWithAllNull; velox::exec::registerVectorFunction( - "row_constructor_with_all_null", + kRowConstructorWithAllNull, std::vector>{}, std::make_unique>(), RowFunctionWithNull::metadata()); velox::exec::registerFunctionCallToSpecialForm( - RowConstructorWithAllNullCallToSpecialForm::kRowConstructorWithAllNull, - std::make_unique()); + kRowConstructorWithAllNull, + std::make_unique(kRowConstructorWithAllNull)); velox::functions::sparksql::registerBitwiseFunctions("spark_"); } } // namespace void registerAllFunctions() { // The registration order matters. Spark sql functions are registered after - // presto sql functions to overwrite the registration for same named functions. + // presto sql functions to overwrite the registration for same named + // functions. velox::functions::prestosql::registerAllScalarFunctions(); velox::functions::sparksql::registerFunctions(""); velox::aggregate::prestosql::registerAllAggregateFunctions( @@ -76,7 +78,8 @@ void registerAllFunctions() { "", true /*registerCompanionFunctions*/, true /*overwrite*/); velox::window::prestosql::registerAllWindowFunctions(); velox::functions::window::sparksql::registerWindowFunctions(""); - // Using function overwrite to handle function names mismatch between Spark and Velox. + // Using function overwrite to handle function names mismatch between Spark + // and Velox. registerFunctionOverwrite(); } diff --git a/cpp/velox/operators/functions/RowConstructorWithAllNull.h b/cpp/velox/operators/functions/RowConstructorWithAllNull.h deleted file mode 100644 index dfc79e1a977b..000000000000 --- a/cpp/velox/operators/functions/RowConstructorWithAllNull.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "RowConstructorWithNull.h" - -namespace gluten { -class RowConstructorWithAllNullCallToSpecialForm : public RowConstructorWithNullCallToSpecialForm { - public: - static constexpr const char* kRowConstructorWithAllNull = "row_constructor_with_all_null"; - - protected: - facebook::velox::exec::ExprPtr constructSpecialForm( - const std::string& name, - const facebook::velox::TypePtr& type, - std::vector&& compiledChildren, - bool trackCpuUsage, - const facebook::velox::core::QueryConfig& config) { - return constructSpecialForm(kRowConstructorWithAllNull, type, std::move(compiledChildren), trackCpuUsage, config); - } -}; -} // namespace gluten diff --git a/cpp/velox/operators/functions/RowConstructorWithNull.cc b/cpp/velox/operators/functions/RowConstructorWithNull.cc index 955d957e26fc..e8b8a288360b 100644 --- a/cpp/velox/operators/functions/RowConstructorWithNull.cc +++ b/cpp/velox/operators/functions/RowConstructorWithNull.cc @@ -32,11 +32,11 @@ facebook::velox::TypePtr RowConstructorWithNullCallToSpecialForm::resolveType( } facebook::velox::exec::ExprPtr RowConstructorWithNullCallToSpecialForm::constructSpecialForm( - const std::string& name, const facebook::velox::TypePtr& type, std::vector&& compiledChildren, bool trackCpuUsage, const facebook::velox::core::QueryConfig& config) { + auto name = this->rowFunctionName; auto [function, metadata] = facebook::velox::exec::vectorFunctionFactories().withRLock( [&config, &name](auto& functionMap) -> std::pair< std::shared_ptr, @@ -52,12 +52,4 @@ facebook::velox::exec::ExprPtr RowConstructorWithNullCallToSpecialForm::construc return std::make_shared( type, std::move(compiledChildren), function, metadata, name, trackCpuUsage); } - -facebook::velox::exec::ExprPtr RowConstructorWithNullCallToSpecialForm::constructSpecialForm( - const facebook::velox::TypePtr& type, - std::vector&& compiledChildren, - bool trackCpuUsage, - const facebook::velox::core::QueryConfig& config) { - return constructSpecialForm(kRowConstructorWithNull, type, std::move(compiledChildren), trackCpuUsage, config); -} } // namespace gluten diff --git a/cpp/velox/operators/functions/RowConstructorWithNull.h b/cpp/velox/operators/functions/RowConstructorWithNull.h index 6cfeaee37a6d..66b745e3ed9b 100644 --- a/cpp/velox/operators/functions/RowConstructorWithNull.h +++ b/cpp/velox/operators/functions/RowConstructorWithNull.h @@ -23,6 +23,10 @@ namespace gluten { class RowConstructorWithNullCallToSpecialForm : public facebook::velox::exec::FunctionCallToSpecialForm { public: + RowConstructorWithNullCallToSpecialForm(const std::string& rowFunctionName) { + this->rowFunctionName = rowFunctionName; + } + facebook::velox::TypePtr resolveType(const std::vector& argTypes) override; facebook::velox::exec::ExprPtr constructSpecialForm( @@ -32,6 +36,7 @@ class RowConstructorWithNullCallToSpecialForm : public facebook::velox::exec::Fu const facebook::velox::core::QueryConfig& config) override; static constexpr const char* kRowConstructorWithNull = "row_constructor_with_null"; + static constexpr const char* kRowConstructorWithAllNull = "row_constructor_with_all_null"; protected: facebook::velox::exec::ExprPtr constructSpecialForm( @@ -40,5 +45,8 @@ class RowConstructorWithNullCallToSpecialForm : public facebook::velox::exec::Fu std::vector&& compiledChildren, bool trackCpuUsage, const facebook::velox::core::QueryConfig& config); + + private: + std::string rowFunctionName; }; } // namespace gluten From 182a0299993b38e04ee053158f71b2500c59eb09 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 14 May 2024 09:43:10 +0800 Subject: [PATCH 066/402] [GLUTEN-5724][VL] Remove redundant counter for calculating VeloxShuffleWriter spill time (#5725) --- cpp/core/shuffle/LocalPartitionWriter.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc b/cpp/core/shuffle/LocalPartitionWriter.cc index b2c4e85b6792..0582ce0e59f6 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.cc +++ b/cpp/core/shuffle/LocalPartitionWriter.cc @@ -44,7 +44,6 @@ class LocalPartitionWriter::LocalSpiller { codec_(codec) {} arrow::Status spill(uint32_t partitionId, std::unique_ptr payload) { - ScopedTimer timer(&spillTime_); // Check spill Type. if (payload->type() != Payload::kUncompressed) { return arrow::Status::Invalid( @@ -317,8 +316,6 @@ class LocalPartitionWriter::PayloadCache { arrow::Result> spill(const std::string& spillFile, arrow::MemoryPool* pool, arrow::util::Codec* codec) { - ScopedTimer timer(&spillTime_); - std::shared_ptr diskSpill = nullptr; ARROW_ASSIGN_OR_RAISE(auto os, arrow::io::FileOutputStream::Open(spillFile, true)); ARROW_ASSIGN_OR_RAISE(auto start, os->Tell()); From ecf61ee2f4bb8fefb4a48635c293cb5d7fe18fa1 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 14 May 2024 13:05:50 +0800 Subject: [PATCH 067/402] [CORE] Add a compilation-time check to forbid case-class inheritance (#5729) --- .../org/apache/spark/sql/delta/DeltaLog.scala | 54 +++++++++++++------ .../org/apache/spark/sql/delta/DeltaLog.scala | 52 ++++++++++++------ .../shuffle/HashPartitioningWrapper.scala | 1 + .../sql/delta/catalog/ClickHouseTableV2.scala | 2 + .../v2/clickhouse/metadata/AddFileTags.scala | 1 + pom.xml | 18 ++++++- 6 files changed, 93 insertions(+), 35 deletions(-) rename {gluten-core => backends-clickhouse}/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala (94%) diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala index 00820a0066d7..0f6455997e56 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala @@ -81,9 +81,10 @@ class DeltaLog private ( with SnapshotManagement with DeltaFileFormat with ReadChecksum { - import org.apache.spark.sql.delta.util.FileNames._ + import DeltaLog._ + implicit private lazy val _clock = clock protected def spark = SparkSession.active @@ -442,8 +443,8 @@ class DeltaLog private ( val fileIndex = TahoeLogFileIndex(spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) - var bucketSpec: Option[BucketSpec] = ClickHouseTableV2.getTable(this).bucketOption - new HadoopFsRelation( + val bucketSpec: Option[BucketSpec] = ClickHouseTableV2.getTable(this).bucketOption + new DeltaHadoopFsRelation( fileIndex, partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata(snapshotToUse.metadata.partitionSchema), @@ -460,20 +461,9 @@ class DeltaLog private ( // conflict with `DeltaLog.options`. snapshotToUse.metadata.format.options ++ options )( - spark - ) with InsertableRelation { - def insert(data: DataFrame, overwrite: Boolean): Unit = { - val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append - WriteIntoDelta( - deltaLog = DeltaLog.this, - mode = mode, - new DeltaOptions(Map.empty[String, String], spark.sessionState.conf), - partitionColumns = Seq.empty, - configuration = Map.empty, - data = data - ).run(spark) - } - } + spark, + this + ) } override def fileFormat(metadata: Metadata = metadata): FileFormat = @@ -482,6 +472,36 @@ class DeltaLog private ( } object DeltaLog extends DeltaLogging { + @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) + private class DeltaHadoopFsRelation( + location: FileIndex, + partitionSchema: StructType, + // The top-level columns in `dataSchema` should match the actual physical file schema, otherwise + // the ORC data source may not work with the by-ordinal mode. + dataSchema: StructType, + bucketSpec: Option[BucketSpec], + fileFormat: FileFormat, + options: Map[String, String])(sparkSession: SparkSession, deltaLog: DeltaLog) + extends HadoopFsRelation( + location, + partitionSchema, + dataSchema, + bucketSpec, + fileFormat, + options)(sparkSession) + with InsertableRelation { + def insert(data: DataFrame, overwrite: Boolean): Unit = { + val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append + WriteIntoDelta( + deltaLog = deltaLog, + mode = mode, + new DeltaOptions(Map.empty[String, String], sparkSession.sessionState.conf), + partitionColumns = Seq.empty, + configuration = Map.empty, + data = data + ).run(sparkSession) + } + } /** * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala index bbf0bdc91c42..4cab6454d15a 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala @@ -85,9 +85,10 @@ class DeltaLog private ( with SnapshotManagement with DeltaFileFormat with ReadChecksum { - import org.apache.spark.sql.delta.util.FileNames._ + import DeltaLog._ + implicit private lazy val _clock = clock protected def spark = SparkSession.active @@ -483,7 +484,7 @@ class DeltaLog private ( val fileIndex = TahoeLogFileIndex(spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) var bucketSpec: Option[BucketSpec] = ClickHouseTableV2.getTable(this).bucketOption - new HadoopFsRelation( + new DeltaHadoopFsRelation( fileIndex, partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata(snapshotToUse.metadata.partitionSchema), @@ -500,20 +501,9 @@ class DeltaLog private ( // conflict with `DeltaLog.options`. snapshotToUse.metadata.format.options ++ options )( - spark - ) with InsertableRelation { - def insert(data: DataFrame, overwrite: Boolean): Unit = { - val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append - WriteIntoDelta( - deltaLog = DeltaLog.this, - mode = mode, - new DeltaOptions(Map.empty[String, String], spark.sessionState.conf), - partitionColumns = Seq.empty, - configuration = Map.empty, - data = data - ).run(spark) - } - } + spark, + this + ) } /** @@ -566,6 +556,36 @@ class DeltaLog private ( } object DeltaLog extends DeltaLogging { + @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) + private class DeltaHadoopFsRelation( + location: FileIndex, + partitionSchema: StructType, + // The top-level columns in `dataSchema` should match the actual physical file schema, otherwise + // the ORC data source may not work with the by-ordinal mode. + dataSchema: StructType, + bucketSpec: Option[BucketSpec], + fileFormat: FileFormat, + options: Map[String, String])(sparkSession: SparkSession, deltaLog: DeltaLog) + extends HadoopFsRelation( + location, + partitionSchema, + dataSchema, + bucketSpec, + fileFormat, + options)(sparkSession) + with InsertableRelation { + def insert(data: DataFrame, overwrite: Boolean): Unit = { + val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append + WriteIntoDelta( + deltaLog = deltaLog, + mode = mode, + new DeltaOptions(Map.empty[String, String], sparkSession.sessionState.conf), + partitionColumns = Seq.empty, + configuration = Map.empty, + data = data + ).run(sparkSession) + } + } /** * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file diff --git a/gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala similarity index 94% rename from gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala index 06ce8fe0f508..bf1cbe4a8046 100644 --- a/gluten-core/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/HashPartitioningWrapper.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning // A wrapper for HashPartitioning to remain original hash expressions. // Only used by CH backend when shuffle hash expressions contains non-field expression. +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class HashPartitioningWrapper( original: Seq[Expression], newExpr: Seq[Expression], diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 8c09ae7c7817..1107c6a2ef76 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -39,6 +39,7 @@ import java.{util => ju} import scala.collection.JavaConverters._ +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class ClickHouseTableV2( override val spark: SparkSession, override val path: Path, @@ -268,6 +269,7 @@ class ClickHouseTableV2( } } +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class TempClickHouseTableV2( override val spark: SparkSession, override val catalogTable: Option[CatalogTable] = None) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala index bdb3a30e914b..0680663eb553 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala @@ -28,6 +28,7 @@ import java.util.{List => JList} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class AddMergeTreeParts( val database: String, val table: String, diff --git a/pom.xml b/pom.xml index dbf46ac176f8..0f37bcbf1851 100644 --- a/pom.xml +++ b/pom.xml @@ -615,13 +615,27 @@ scala-maven-plugin ${scala.compiler.version} + + + org.wartremover + wartremover_${scala.binary.version} + 3.1.6 + + + + + io.github.zhztheplayer.scalawarts + scalawarts + 0.1.1 + + ${scala.recompile.mode} - -Wconf:msg=While parsing annotations in:silent + -Wconf:msg=While parsing annotations in:silent,any:e -Ywarn-unused:imports - -Xfatal-warnings -deprecation -feature + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass From 4907f254d7cb4508f6fccb56fe9ba7ee617bb0b7 Mon Sep 17 00:00:00 2001 From: Terry Wang Date: Tue, 14 May 2024 16:45:25 +0800 Subject: [PATCH 068/402] [GLUTEN-5739][VL] Fix ShuffleReaderMetrics deserializeTime always is zero (#5738) --- .../java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java index c4548365d2e7..f6dfadaafcf6 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java @@ -37,7 +37,7 @@ public long getIpcTime() { return ipcTime; } - public void setDeserializeTime(long ipcTime) { + public void setDeserializeTime(long deserializeTime) { this.deserializeTime = deserializeTime; } From e80785625c345271ecb279540468d74bdcafd394 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Tue, 14 May 2024 17:54:57 +0800 Subject: [PATCH 069/402] [GLUTEN-5620][CORE] Remove check_overflow and refactor code (#5654) --- .../clickhouse/CHTransformerApi.scala | 1 + .../backendsapi/velox/VeloxBackend.scala | 4 +- .../velox/VeloxTransformerApi.scala | 9 +- .../backendsapi/BackendSettingsApi.scala | 4 +- .../gluten/backendsapi/TransformerApi.scala | 3 +- .../expression/ExpressionConverter.scala | 71 ++++----- .../UnaryExpressionTransformer.scala | 2 + .../gluten/utils/DecimalArithmeticUtil.scala | 143 ++++-------------- .../spark/sql/utils/DecimalTypeUtil.scala | 26 ++++ 9 files changed, 104 insertions(+), 159 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/spark/sql/utils/DecimalTypeUtil.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index df1ca9c6817a..ee46d685c6c1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -219,6 +219,7 @@ class CHTransformerApi extends TransformerApi with Logging { args: java.lang.Object, substraitExprName: String, childNode: ExpressionNode, + childResultType: DataType, dataType: DecimalType, nullable: Boolean, nullOnOverflow: Boolean): ExpressionNode = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 5509d37e8eb1..c16b3624f319 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -449,14 +449,12 @@ object VeloxBackendSettings extends BackendSettingsApi { override def fallbackAggregateWithEmptyOutputChild(): Boolean = true override def recreateJoinExecOnFallback(): Boolean = true - override def rescaleDecimalLiteral(): Boolean = true + override def rescaleDecimalArithmetic(): Boolean = true /** Get the config prefix for each backend */ override def getBackendConfigPrefix(): String = GlutenConfig.GLUTEN_CONFIG_PREFIX + VeloxBackend.BACKEND_NAME - override def rescaleDecimalIntegralExpression(): Boolean = true - override def shuffleSupportedCodec(): Set[String] = SHUFFLE_SUPPORTED_CODEC override def resolveNativeConf(nativeConf: java.util.Map[String, String]): Unit = {} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index e5aa281a8e79..33f612440883 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -80,11 +80,16 @@ class VeloxTransformerApi extends TransformerApi with Logging { args: java.lang.Object, substraitExprName: String, childNode: ExpressionNode, + childResultType: DataType, dataType: DecimalType, nullable: Boolean, nullOnOverflow: Boolean): ExpressionNode = { - val typeNode = ConverterUtils.getTypeNode(dataType, nullable) - ExpressionBuilder.makeCast(typeNode, childNode, !nullOnOverflow) + if (childResultType.equals(dataType)) { + childNode + } else { + val typeNode = ConverterUtils.getTypeNode(dataType, nullable) + ExpressionBuilder.makeCast(typeNode, childNode, !nullOnOverflow) + } } override def getNativePlanString(substraitPlan: Array[Byte], details: Boolean): String = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index c8729561dfe0..9c5c13271aeb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -85,7 +85,7 @@ trait BackendSettingsApi { def supportShuffleWithProject(outputPartitioning: Partitioning, child: SparkPlan): Boolean = false def utilizeShuffledHashJoinHint(): Boolean = false def excludeScanExecFromCollapsedStage(): Boolean = false - def rescaleDecimalLiteral: Boolean = false + def rescaleDecimalArithmetic: Boolean = false /** * Whether to replace sort agg with hash agg., e.g., sort agg will be used in spark's planning for @@ -106,8 +106,6 @@ trait BackendSettingsApi { */ def transformCheckOverflow: Boolean = true - def rescaleDecimalIntegralExpression(): Boolean = false - def shuffleSupportedCodec(): Set[String] def needOutputSchemaForPlan(): Boolean = false diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala index 49a97a8a4d06..7a10dc68c8aa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.types.DecimalType +import org.apache.spark.sql.types.{DataType, DecimalType} import org.apache.spark.util.collection.BitSet import com.google.protobuf.{Any, Message} @@ -69,6 +69,7 @@ trait TransformerApi { args: java.lang.Object, substraitExprName: String, childNode: ExpressionNode, + childResultType: DataType, dataType: DecimalType, nullable: Boolean, nullOnOverflow: Boolean): ExpressionNode diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 495fbf8d5a4b..b7b946268ff5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -101,6 +101,28 @@ object ExpressionConverter extends SQLConfHelper with Logging { } } + private def genRescaleDecimalTransformer( + substraitName: String, + b: BinaryArithmetic, + attributeSeq: Seq[Attribute], + expressionsMap: Map[Class[_], String]): DecimalArithmeticExpressionTransformer = { + val rescaleBinary = DecimalArithmeticUtil.rescaleLiteral(b) + val (left, right) = DecimalArithmeticUtil.rescaleCastForDecimal( + DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.left), + DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.right)) + val resultType = DecimalArithmeticUtil.getResultType( + b, + left.dataType.asInstanceOf[DecimalType], + right.dataType.asInstanceOf[DecimalType] + ) + + val leftChild = + replaceWithExpressionTransformerInternal(left, attributeSeq, expressionsMap) + val rightChild = + replaceWithExpressionTransformerInternal(right, attributeSeq, expressionsMap) + DecimalArithmeticExpressionTransformer(substraitName, leftChild, rightChild, resultType, b) + } + private def replaceWithExpressionTransformerInternal( expr: Expression, attributeSeq: Seq[Attribute], @@ -492,7 +514,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { expr.children.map( replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), expr) - case CheckOverflow(b: BinaryArithmetic, decimalType, _) if !BackendsApiManager.getSettings.transformCheckOverflow && DecimalArithmeticUtil.isDecimalArithmetic(b) => @@ -507,55 +528,25 @@ object ExpressionConverter extends SQLConfHelper with Logging { rightChild, decimalType, b) - case c: CheckOverflow => CheckOverflowTransformer( substraitExprName, replaceWithExpressionTransformerInternal(c.child, attributeSeq, expressionsMap), + c.child.dataType, c) - case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b) => DecimalArithmeticUtil.checkAllowDecimalArithmetic() if (!BackendsApiManager.getSettings.transformCheckOverflow) { - val leftChild = - replaceWithExpressionTransformerInternal(b.left, attributeSeq, expressionsMap) - val rightChild = - replaceWithExpressionTransformerInternal(b.right, attributeSeq, expressionsMap) - DecimalArithmeticExpressionTransformer( + GenericExpressionTransformer( substraitExprName, - leftChild, - rightChild, - b.dataType.asInstanceOf[DecimalType], - b) - } else { - val rescaleBinary = if (BackendsApiManager.getSettings.rescaleDecimalLiteral) { - DecimalArithmeticUtil.rescaleLiteral(b) - } else { - b - } - val (left, right) = DecimalArithmeticUtil.rescaleCastForDecimal( - DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.left), - DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.right)) - val leftChild = - replaceWithExpressionTransformerInternal(left, attributeSeq, expressionsMap) - val rightChild = - replaceWithExpressionTransformerInternal(right, attributeSeq, expressionsMap) - - val resultType = DecimalArithmeticUtil.getResultTypeForOperation( - DecimalArithmeticUtil.getOperationType(b), - DecimalArithmeticUtil - .getResultType(leftChild) - .getOrElse(left.dataType.asInstanceOf[DecimalType]), - DecimalArithmeticUtil - .getResultType(rightChild) - .getOrElse(right.dataType.asInstanceOf[DecimalType]) + expr.children.map( + replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), + expr ) - DecimalArithmeticExpressionTransformer( - substraitExprName, - leftChild, - rightChild, - resultType, - b) + } else { + // Without the rescale and remove cast, result is right for high version Spark, + // but performance regression in velox + genRescaleDecimalTransformer(substraitExprName, b, attributeSeq, expressionsMap) } case n: NaNvl => BackendsApiManager.getSparkPlanExecApiInstance.genNaNvlTransformer( diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala index 88df12b84d9a..2d3840ce4f03 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala @@ -152,6 +152,7 @@ case class PosExplodeTransformer( case class CheckOverflowTransformer( substraitExprName: String, child: ExpressionTransformer, + childResultType: DataType, original: CheckOverflow) extends ExpressionTransformer { @@ -160,6 +161,7 @@ case class CheckOverflowTransformer( args, substraitExprName, child.doTransform(args), + childResultType, original.dataType, original.nullable, original.nullOnOverflow) diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala index 148cc4e609ce..479eb8bb5c29 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala @@ -18,69 +18,40 @@ package org.apache.gluten.utils import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.{CheckOverflowTransformer, ChildTransformer, DecimalArithmeticExpressionTransformer, ExpressionTransformer} import org.apache.gluten.expression.ExpressionConverter.conf import org.apache.spark.sql.catalyst.analysis.DecimalPrecision import org.apache.spark.sql.catalyst.expressions.{Add, BinaryArithmetic, Cast, Divide, Expression, Literal, Multiply, Pmod, PromotePrecision, Remainder, Subtract} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ByteType, Decimal, DecimalType, IntegerType, LongType, ShortType} - -import scala.annotation.tailrec +import org.apache.spark.sql.utils.DecimalTypeUtil object DecimalArithmeticUtil { - object OperationType extends Enumeration { - type Config = Value - val ADD, SUBTRACT, MULTIPLY, DIVIDE, MOD = Value - } - - private val MIN_ADJUSTED_SCALE = 6 - val MAX_PRECISION = 38 - // Returns the result decimal type of a decimal arithmetic computing. - def getResultTypeForOperation( - operationType: OperationType.Config, - type1: DecimalType, - type2: DecimalType): DecimalType = { + def getResultType(expr: BinaryArithmetic, type1: DecimalType, type2: DecimalType): DecimalType = { var resultScale = 0 var resultPrecision = 0 - operationType match { - case OperationType.ADD => + expr match { + case _: Add => resultScale = Math.max(type1.scale, type2.scale) resultPrecision = resultScale + Math.max(type1.precision - type1.scale, type2.precision - type2.scale) + 1 - case OperationType.SUBTRACT => + case _: Subtract => resultScale = Math.max(type1.scale, type2.scale) resultPrecision = resultScale + Math.max(type1.precision - type1.scale, type2.precision - type2.scale) + 1 - case OperationType.MULTIPLY => + case _: Multiply => resultScale = type1.scale + type2.scale resultPrecision = type1.precision + type2.precision + 1 - case OperationType.DIVIDE => - resultScale = Math.max(MIN_ADJUSTED_SCALE, type1.scale + type2.precision + 1) + case _: Divide => + resultScale = + Math.max(DecimalType.MINIMUM_ADJUSTED_SCALE, type1.scale + type2.precision + 1) resultPrecision = type1.precision - type1.scale + type2.scale + resultScale - case OperationType.MOD => - resultScale = Math.max(type1.scale, type2.scale) - resultPrecision = - Math.min(type1.precision - type1.scale, type2.precision - type2.scale + resultScale) case other => throw new GlutenNotSupportException(s"$other is not supported.") } - adjustScaleIfNeeded(resultPrecision, resultScale) - } - - // Returns the adjusted decimal type when the precision is larger the maximum. - private def adjustScaleIfNeeded(precision: Int, scale: Int): DecimalType = { - var typePrecision = precision - var typeScale = scale - if (precision > MAX_PRECISION) { - val minScale = Math.min(scale, MIN_ADJUSTED_SCALE) - val delta = precision - MAX_PRECISION - typePrecision = MAX_PRECISION - typeScale = Math.max(scale - delta, minScale) - } - DecimalType(typePrecision, typeScale) + DecimalTypeUtil.adjustPrecisionScale(resultPrecision, resultScale) } // If casting between DecimalType, unnecessary cast is skipped to avoid data loss, @@ -98,18 +69,6 @@ object DecimalArithmeticUtil { } else false } - // Returns the operation type of a binary arithmetic expression. - def getOperationType(b: BinaryArithmetic): OperationType.Config = { - b match { - case _: Add => OperationType.ADD - case _: Subtract => OperationType.SUBTRACT - case _: Multiply => OperationType.MULTIPLY - case _: Divide => OperationType.DIVIDE - case other => - throw new GlutenNotSupportException(s"$other is not supported.") - } - } - // For decimal * 10 case, dec will be Decimal(38, 18), then the result precision is wrong, // so here we will get the real precision and scale of the literal. private def getNewPrecisionScale(dec: Decimal): (Integer, Integer) = { @@ -179,9 +138,7 @@ object DecimalArithmeticUtil { if (isWiderType) (e1, newE2) else (e1, e2) } - if (!BackendsApiManager.getSettings.rescaleDecimalIntegralExpression()) { - (left, right) - } else if (!isPromoteCast(left) && isPromoteCastIntegral(right)) { + if (!isPromoteCast(left) && isPromoteCastIntegral(right)) { // Have removed PromotePrecision(Cast(DecimalType)). // Decimal * cast int. doScale(left, right) @@ -202,66 +159,32 @@ object DecimalArithmeticUtil { * @return * expression removed child PromotePrecision->Cast */ - def removeCastForDecimal(arithmeticExpr: Expression): Expression = { - arithmeticExpr match { - case precision: PromotePrecision => - precision.child match { - case cast: Cast - if cast.dataType.isInstanceOf[DecimalType] - && cast.child.dataType.isInstanceOf[DecimalType] => - cast.child - case _ => arithmeticExpr - } - case _ => arithmeticExpr - } + def removeCastForDecimal(arithmeticExpr: Expression): Expression = arithmeticExpr match { + case PromotePrecision(_ @Cast(child, _: DecimalType, _, _)) + if child.dataType.isInstanceOf[DecimalType] => + child + case _ => arithmeticExpr } - @tailrec - def getResultType(transformer: ExpressionTransformer): Option[DecimalType] = { - transformer match { - case ChildTransformer(child) => - getResultType(child) - case CheckOverflowTransformer(_, _, original) => - Some(original.dataType) - case DecimalArithmeticExpressionTransformer(_, _, _, resultType, _) => - Some(resultType) - case _ => None - } - } - - private def isPromoteCastIntegral(expr: Expression): Boolean = { - expr match { - case precision: PromotePrecision => - precision.child match { - case cast: Cast if cast.dataType.isInstanceOf[DecimalType] => - cast.child.dataType match { - case IntegerType | ByteType | ShortType | LongType => true - case _ => false - } - case _ => false - } - case _ => false - } + private def isPromoteCastIntegral(expr: Expression): Boolean = expr match { + case PromotePrecision(_ @Cast(child, _: DecimalType, _, _)) => + child.dataType match { + case IntegerType | ByteType | ShortType | LongType => true + case _ => false + } + case _ => false } - private def rescaleCastForOneSide(expr: Expression): Expression = { - expr match { - case precision: PromotePrecision => - precision.child match { - case castInt: Cast - if castInt.dataType.isInstanceOf[DecimalType] && - BackendsApiManager.getSettings.rescaleDecimalIntegralExpression() => - castInt.child.dataType match { - case IntegerType | ByteType | ShortType => - precision.withNewChildren(Seq(Cast(castInt.child, DecimalType(10, 0)))) - case LongType => - precision.withNewChildren(Seq(Cast(castInt.child, DecimalType(20, 0)))) - case _ => expr - } - case _ => expr - } - case _ => expr - } + private def rescaleCastForOneSide(expr: Expression): Expression = expr match { + case precision @ PromotePrecision(_ @Cast(child, _: DecimalType, _, _)) => + child.dataType match { + case IntegerType | ByteType | ShortType => + precision.withNewChildren(Seq(Cast(child, DecimalType(10, 0)))) + case LongType => + precision.withNewChildren(Seq(Cast(child, DecimalType(20, 0)))) + case _ => expr + } + case _ => expr } private def checkIsWiderType( diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/utils/DecimalTypeUtil.scala b/gluten-core/src/main/scala/org/apache/spark/sql/utils/DecimalTypeUtil.scala new file mode 100644 index 000000000000..f7334bcb2382 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/sql/utils/DecimalTypeUtil.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.utils + +import org.apache.spark.sql.types.DecimalType + +object DecimalTypeUtil { + def adjustPrecisionScale(precision: Int, scale: Int): DecimalType = { + DecimalType.adjustPrecisionScale(precision, scale) + } + +} From c7306128a1d5808521c4b15931f53b939bb84f51 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Tue, 14 May 2024 11:13:03 -0500 Subject: [PATCH 070/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240514) (#5732) Co-authored-by: kyligence-git We need merge due to ClickHouse/ClickHouse#60469 --- cpp-ch/clickhouse.version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index aba4b7c567bd..7e5d2a5b97f5 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240511 -CH_COMMIT=725c7a6c78e \ No newline at end of file +CH_BRANCH=rebase_ch/20240514 +CH_COMMIT=c0d37f6fa5b \ No newline at end of file From 3888f45e060c4d8121c38190621282780b77a032 Mon Sep 17 00:00:00 2001 From: James Xu Date: Wed, 15 May 2024 02:20:43 +0800 Subject: [PATCH 071/402] [GLUTEN-5745][VL] Add more comments for GenerateRel conversion logic (#5746) Add more comments for GenerateRel conversion logic --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 6ee7ad8278dd..366ab5abdc9d 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -733,8 +733,14 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: auto projNode = std::dynamic_pointer_cast(childNode); if (projNode != nullptr && projNode->names().size() > requiredChildOutput.size()) { - // generator is a scalarfunction node -> explode(array(col, 'all')) - // use the last one, this is ensure by scala code + // Generator function's input is not a field reference, e.g. explode(array(1,2,3)), a sample + // input substrait plan is like the following(the plan structure is ensured by scala code): + // + // Generate explode([1,2,3] AS _pre_0#129), false, [col#126] + // +- Project [fake_column#128, [1,2,3] AS _pre_0#129] + // +- RewrittenNodeWall Scan OneRowRelation[fake_column#128] + // + // The last projection column in GeneratorRel's child(Project) is the column we need to unnest auto innerName = projNode->names().back(); auto innerExpr = projNode->projections().back(); @@ -743,9 +749,12 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: VELOX_CHECK_NOT_NULL(unnestFieldExpr, " the key in unnest Operator only support field"); unnest.emplace_back(unnestFieldExpr); } else { - // generator should be a array column -> explode(col) - auto explodeFunc = generator.scalar_function(); - auto unnestExpr = exprConverter_->toVeloxExpr(explodeFunc.arguments(0).value(), inputType); + // Generator function's input is a field reference, e.g. explode(col), generator + // function's first argument is the field reference we need to unnest. + // This assumption holds for all the supported generator function: + // explode, posexplode, inline. + auto generatorFunc = generator.scalar_function(); + auto unnestExpr = exprConverter_->toVeloxExpr(generatorFunc.arguments(0).value(), inputType); auto unnestFieldExpr = std::dynamic_pointer_cast(unnestExpr); VELOX_CHECK_NOT_NULL(unnestFieldExpr, " the key in unnest Operator only support field"); unnest.emplace_back(unnestFieldExpr); From 37be4aa026c0aa9265a89fb7ae6b4210a1eaafed Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 15 May 2024 02:35:43 +0800 Subject: [PATCH 072/402] [VL] Daily Update Velox Version (2024_05_14) (#5733) Auto update success --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index b5c8fe5dcf35..558de0288f02 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_12 +VELOX_BRANCH=2024_05_14 VELOX_HOME="" #Set on run gluten on HDFS From ba64f4c227c02ea86ef29c46d63194c297784f52 Mon Sep 17 00:00:00 2001 From: Tengfei Huang Date: Wed, 15 May 2024 13:27:35 +0800 Subject: [PATCH 073/402] [VL] Drop the test table after all tests in FallbackSuite (#5737) --- .../test/scala/org/apache/gluten/execution/FallbackSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index e8833a43ca39..15a71ceb587b 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -62,6 +62,7 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl override protected def afterAll(): Unit = { spark.sql("drop table tmp1") spark.sql("drop table tmp2") + spark.sql("drop table tmp3") super.afterAll() } From 36c60044a52fbc36519cca317c362462dbb37173 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Wed, 15 May 2024 00:32:41 -0500 Subject: [PATCH 074/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240515) (#5747) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240515) * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/63432 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- cpp-ch/local-engine/Parser/JoinRelParser.cpp | 3 ++- cpp-ch/local-engine/tests/benchmark_local_engine.cpp | 6 +++--- cpp-ch/local-engine/tests/gtest_ch_join.cpp | 3 ++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 7e5d2a5b97f5..6a58dce0f4c8 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240514 -CH_COMMIT=c0d37f6fa5b \ No newline at end of file +CH_BRANCH=rebase_ch/20240515 +CH_COMMIT=00867009134 \ No newline at end of file diff --git a/cpp-ch/local-engine/Parser/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/JoinRelParser.cpp index 023a51552f82..8f7f35d5ef23 100644 --- a/cpp-ch/local-engine/Parser/JoinRelParser.cpp +++ b/cpp-ch/local-engine/Parser/JoinRelParser.cpp @@ -135,7 +135,8 @@ std::pair getJoinKindAndStrictness(substrait:: std::shared_ptr createDefaultTableJoin(substrait::JoinRel_JoinType join_type) { auto & global_context = SerializedPlanParser::global_context; - auto table_join = std::make_shared(global_context->getSettings(), global_context->getGlobalTemporaryVolume()); + auto table_join = std::make_shared( + global_context->getSettings(), global_context->getGlobalTemporaryVolume(), global_context->getTempDataOnDisk()); std::pair kind_and_strictness = getJoinKindAndStrictness(join_type); table_join->setKind(kind_and_strictness.first); diff --git a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp index 51ee7ad12f47..89fa4fa961ea 100644 --- a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp +++ b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp @@ -51,11 +51,10 @@ #include #include #include -#include #include "testConfig.h" #if defined(__SSE2__) -# include +#include #endif @@ -836,7 +835,8 @@ QueryPlanPtr readFromMergeTree(MergeTreeWithSnapshot storage) QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr right, String left_key, String right_key, size_t block_size = 8192) { - auto join = std::make_shared(global_context->getSettings(), global_context->getGlobalTemporaryVolume()); + auto join = std::make_shared( + global_context->getSettings(), global_context->getGlobalTemporaryVolume(), global_context->getTempDataOnDisk()); auto left_columns = left->getCurrentDataStream().header.getColumnsWithTypeAndName(); auto right_columns = right->getCurrentDataStream().header.getColumnsWithTypeAndName(); join->setKind(JoinKind::Left); diff --git a/cpp-ch/local-engine/tests/gtest_ch_join.cpp b/cpp-ch/local-engine/tests/gtest_ch_join.cpp index 1621a9e8868d..739390302b46 100644 --- a/cpp-ch/local-engine/tests/gtest_ch_join.cpp +++ b/cpp-ch/local-engine/tests/gtest_ch_join.cpp @@ -85,7 +85,8 @@ TEST(TestJoin, simple) QueryPlan right_plan; right_plan.addStep(std::make_unique(Pipe(right_table))); - auto join = std::make_shared(global_context->getSettings(), global_context->getGlobalTemporaryVolume()); + auto join = std::make_shared( + global_context->getSettings(), global_context->getGlobalTemporaryVolume(), global_context->getTempDataOnDisk()); join->setKind(JoinKind::Left); join->setStrictness(JoinStrictness::All); join->setColumnsFromJoinedTable(right.getNamesAndTypesList()); From 3b8ea67f49d0b3afdbc327905a6cb0fefe23b56a Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 15 May 2024 13:56:51 +0800 Subject: [PATCH 075/402] [VL] CI: Gluten-it: Fix unreadable test reporting when there are query failures (#5753) --- .../apache/gluten/integration/tpc/action/Queries.scala | 2 +- .../gluten/integration/tpc/action/QueriesCompare.scala | 2 +- .../gluten/integration/tpc/action/TableFormatter.scala | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala index edeb960fcba9..290b8e3f5b0c 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala @@ -80,7 +80,7 @@ case class Queries( var all = Queries.aggregate(results, "all") if (passedCount != count) { - all = Queries.aggregate(succeed, "all succeed") ::: all + all = Queries.aggregate(succeed, "succeeded") ::: all } println("Overall: ") diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala index cfb3e7dc5378..404d75cb426e 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala @@ -80,7 +80,7 @@ case class QueriesCompare( var all = QueriesCompare.aggregate(results, "all") if (passedCount != count) { - all = QueriesCompare.aggregate(succeed, "all succeed") ::: all + all = QueriesCompare.aggregate(succeed, "succeeded") ::: all } println("Overall: ") diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala index cb6ab7ebd056..8aeea9938e90 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala @@ -46,6 +46,12 @@ object TableFormatter { } override def print(s: OutputStream): Unit = { + val printer = new PrintStream(s) + if (rows.isEmpty) { + printer.println("(N/A)") + printer.flush() + return + } val numFields = schema.fields.size val widths = (0 until numFields) .map { i => @@ -58,13 +64,11 @@ object TableFormatter { pBuilder ++= s"%${w}s|" } val pattern = pBuilder.toString() - val printer = new PrintStream(s) printer.println(String.format(pattern, schema.fields: _*)) rows.foreach { r => printer.println(String.format(pattern, r: _*)) } printer.flush() - printer.close() } } From a7e4435b152e318c2f529dd9f1e8c46082f3e214 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 15 May 2024 14:12:38 +0800 Subject: [PATCH 076/402] [VL] Fix build script in Alinux3 (#5749) --- ep/build-velox/src/get_velox.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 558de0288f02..d1e6054d810c 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -199,13 +199,14 @@ function process_setup_centos7 { function process_setup_alinux3 { process_setup_centos8 sed -i "s/.*dnf_install epel-release/#&/" scripts/setup-centos8.sh + sed -i "s/.*run_and_time install_conda/#&/" scripts/setup-centos8.sh sed -i "s/.*dnf config-manager --set-enabled powertools/#&/" scripts/setup-centos8.sh sed -i "s/gcc-toolset-9 //" scripts/setup-centos8.sh sed -i "s/.*source \/opt\/rh\/gcc-toolset-9\/enable/#&/" scripts/setup-centos8.sh sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|' scripts/setup-centos8.sh sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|' scripts/setup-centos8.sh sed -i 's/python39 python39-devel python39-pip //g' scripts/setup-centos8.sh - sed -i 's/pip3.9/pip3.6/g' scripts/setup-centos8.sh + sed -i "s/.*pip.* install/#&/" scripts/setup-centos8.sh sed -i 's/ADDITIONAL_FLAGS=""/ADDITIONAL_FLAGS="-Wno-stringop-overflow"/g' scripts/setup-helper-functions.sh sed -i "s/\${CMAKE_INSTALL_LIBDIR}/lib64/" third_party/CMakeLists.txt } From a53ecc4a1261afe52c659a4333af20c687552f0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Wed, 15 May 2024 14:35:20 +0800 Subject: [PATCH 077/402] [VL] Enable GlutenParqutRowIndexSuite for Spark 3.4/3.5 (#5740) --- .../execution/BatchScanExecTransformer.scala | 5 + .../utils/velox/VeloxTestSettings.scala | 8 +- .../parquet/GlutenParquetRowIndexSuite.scala | 335 ++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 5 +- .../parquet/GlutenParquetRowIndexSuite.scala | 342 +++++++++++++++++- .../datasources/v2/BatchScanExecShim.scala | 7 + .../datasources/v2/BatchScanExecShim.scala | 14 + .../execution/FileSourceScanExecShim.scala | 5 +- .../datasources/v2/BatchScanExecShim.scala | 20 + .../execution/FileSourceScanExecShim.scala | 5 +- .../datasources/v2/BatchScanExecShim.scala | 20 + 11 files changed, 756 insertions(+), 10 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala index 3aeeffae1791..b0c8c59e7bb5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala @@ -136,6 +136,11 @@ abstract class BatchScanExecTransformerBase( if (pushedAggregate.nonEmpty) { return ValidationResult.notOk(s"Unsupported aggregation push down for $scan.") } + + if (hasUnsupportedColumns) { + return ValidationResult.notOk(s"Unsupported columns scan in native.") + } + super.doValidateInternal() } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 498ed5ef4da4..1afa203ab6f5 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.execution.datasources.csv.{GlutenCSVLegacyTimeParser import org.apache.spark.sql.execution.datasources.exchange.GlutenValidateRequirementsSuite import org.apache.spark.sql.execution.datasources.json.{GlutenJsonLegacyTimeParserSuite, GlutenJsonV1Suite, GlutenJsonV2Suite} import org.apache.spark.sql.execution.datasources.orc.{GlutenOrcColumnarBatchReaderSuite, GlutenOrcFilterSuite, GlutenOrcPartitionDiscoverySuite, GlutenOrcSourceSuite, GlutenOrcV1FilterSuite, GlutenOrcV1PartitionDiscoverySuite, GlutenOrcV1QuerySuite, GlutenOrcV1SchemaPruningSuite, GlutenOrcV2QuerySuite, GlutenOrcV2SchemaPruningSuite} -import org.apache.spark.sql.execution.datasources.parquet.{GlutenParquetColumnIndexSuite, GlutenParquetCompressionCodecPrecedenceSuite, GlutenParquetDeltaByteArrayEncodingSuite, GlutenParquetDeltaEncodingInteger, GlutenParquetDeltaEncodingLong, GlutenParquetDeltaLengthByteArrayEncodingSuite, GlutenParquetEncodingSuite, GlutenParquetFieldIdIOSuite, GlutenParquetFileFormatV1Suite, GlutenParquetFileFormatV2Suite, GlutenParquetInteroperabilitySuite, GlutenParquetIOSuite, GlutenParquetProtobufCompatibilitySuite, GlutenParquetRebaseDatetimeV1Suite, GlutenParquetRebaseDatetimeV2Suite, GlutenParquetSchemaInferenceSuite, GlutenParquetSchemaSuite, GlutenParquetThriftCompatibilitySuite, GlutenParquetV1FilterSuite, GlutenParquetV1PartitionDiscoverySuite, GlutenParquetV1QuerySuite, GlutenParquetV1SchemaPruningSuite, GlutenParquetV2FilterSuite, GlutenParquetV2PartitionDiscoverySuite, GlutenParquetV2QuerySuite, GlutenParquetV2SchemaPruningSuite, GlutenParquetVectorizedSuite} +import org.apache.spark.sql.execution.datasources.parquet.{GlutenParquetColumnIndexSuite, GlutenParquetCompressionCodecPrecedenceSuite, GlutenParquetDeltaByteArrayEncodingSuite, GlutenParquetDeltaEncodingInteger, GlutenParquetDeltaEncodingLong, GlutenParquetDeltaLengthByteArrayEncodingSuite, GlutenParquetEncodingSuite, GlutenParquetFieldIdIOSuite, GlutenParquetFileFormatV1Suite, GlutenParquetFileFormatV2Suite, GlutenParquetInteroperabilitySuite, GlutenParquetIOSuite, GlutenParquetProtobufCompatibilitySuite, GlutenParquetRebaseDatetimeV1Suite, GlutenParquetRebaseDatetimeV2Suite, GlutenParquetRowIndexSuite, GlutenParquetSchemaInferenceSuite, GlutenParquetSchemaSuite, GlutenParquetThriftCompatibilitySuite, GlutenParquetV1FilterSuite, GlutenParquetV1PartitionDiscoverySuite, GlutenParquetV1QuerySuite, GlutenParquetV1SchemaPruningSuite, GlutenParquetV2FilterSuite, GlutenParquetV2PartitionDiscoverySuite, GlutenParquetV2QuerySuite, GlutenParquetV2SchemaPruningSuite, GlutenParquetVectorizedSuite} import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite} import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite} import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite @@ -1189,9 +1189,9 @@ class VeloxTestSettings extends BackendTestSettings { // Row index metadata column support in Velox isn't ready yet, refer velox-9147 .exclude("reading _tmp_metadata_row_index - not present in a table") .exclude("reading _tmp_metadata_row_index - present in a table") - // Row index metadata column support in Velox isn't ready yet, refer velox-9147 - // enableSuite[GlutenParquetRowIndexSuite] - + enableSuite[GlutenParquetRowIndexSuite] + .excludeByPrefix("row index generation") + .excludeByPrefix("invalid row index column type") override def getSQLQueryTestSettings: SQLQueryTestSettings = VeloxSQLQueryTestSettings } // scalastyle:on line.size.limit diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala index acf6a2b6384d..6f153450cb96 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -16,6 +16,339 @@ */ package org.apache.spark.sql.execution.datasources.parquet +import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer} + import org.apache.spark.sql.GlutenSQLTestsBaseTrait +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2 +import org.apache.spark.sql.functions.{col, max, min} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{LongType, StringType} + +import org.apache.hadoop.fs.Path +import org.apache.parquet.column.ParquetProperties._ +import org.apache.parquet.format.converter.ParquetMetadataConverter +import org.apache.parquet.hadoop.ParquetOutputFormat +import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE + +import java.io.File + +import scala.collection.JavaConverters._ + +class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait { + import testImplicits._ + override def beforeAll(): Unit = { + super.beforeAll() + sparkContext.setLogLevel("info") + } + + private def readRowGroupRowCounts(path: String): Seq[Long] = { + ParquetFooterReader + .readFooter( + spark.sessionState.newHadoopConf(), + new Path(path), + ParquetMetadataConverter.NO_FILTER) + .getBlocks + .asScala + .map(_.getRowCount) + } + + private def readRowGroupRowCounts(dir: File): Seq[Seq[Long]] = { + assert(dir.isDirectory) + dir + .listFiles() + .filter(f => f.isFile && f.getName.endsWith("parquet")) + .map(f => readRowGroupRowCounts(f.getAbsolutePath)) + } + + /** Do the files contain exactly one row group? */ + private def assertOneRowGroup(dir: File): Unit = { + readRowGroupRowCounts(dir).foreach { + rcs => assert(rcs.length == 1, "expected one row group per file") + } + } + + /** + * Do the files have a good layout to test row group skipping (both range metadata filter, and by + * using min/max). + */ + private def assertTinyRowGroups(dir: File): Unit = { + readRowGroupRowCounts(dir).foreach { + rcs => + assert(rcs.length > 1, "expected multiple row groups per file") + assert(rcs.last <= DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK) + assert( + rcs.reverse.tail.distinct == Seq(DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK), + "expected row groups with minimal row count") + } + } + + /** + * Do the files have a good layout to test a combination of page skipping and row group skipping? + */ + private def assertIntermediateRowGroups(dir: File): Unit = { + readRowGroupRowCounts(dir).foreach { + rcs => + assert(rcs.length >= 3, "expected at least 3 row groups per file") + rcs.reverse.tail.foreach { + rc => + assert( + rc > DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK, + "expected row groups larger than minimal row count") + } + } + } + + case class GlutenRowIndexTestConf( + numRows: Long = 10000L, + useMultipleFiles: Boolean = false, + useVectorizedReader: Boolean = true, + useSmallPages: Boolean = false, + useSmallRowGroups: Boolean = false, + useSmallSplits: Boolean = false, + useFilter: Boolean = false, + useDataSourceV2: Boolean = false) { + + val NUM_MULTIPLE_FILES = 4 + // The test doesn't work correctly if the number of records per file is uneven. + assert(!useMultipleFiles || (numRows % NUM_MULTIPLE_FILES == 0)) + + def numFiles: Int = if (useMultipleFiles) { NUM_MULTIPLE_FILES } + else { 1 } + + def rowGroupSize: Long = if (useSmallRowGroups) { + if (useSmallPages) { + // Each file will contain multiple row groups. All of them (except for the last one) + // will contain more than DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK, so that individual + // pages within the row group can be skipped. + 2048L + } else { + // Each file will contain multiple row groups. All of them (except for the last one) + // will contain exactly DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK records. + 64L + } + } else { + // Each file will contain a single row group. + DEFAULT_BLOCK_SIZE + } + + def pageSize: Long = if (useSmallPages) { + // Each page (except for the last one for each column) will contain exactly + // DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK records. + 64L + } else { + DEFAULT_PAGE_SIZE + } + + def writeFormat: String = "parquet" + def readFormat: String = if (useDataSourceV2) { + classOf[ParquetDataSourceV2].getCanonicalName + } else { + "parquet" + } + + assert(useSmallRowGroups || !useSmallSplits) + def filesMaxPartitionBytes: Long = if (useSmallSplits) { + 256L + } else { + SQLConf.FILES_MAX_PARTITION_BYTES.defaultValue.get + } + + def desc: String = { + { if (useVectorizedReader) Seq("vectorized reader") else Seq("parquet-mr reader") } ++ { + if (useMultipleFiles) Seq("many files") else Seq.empty[String] + } ++ { if (useFilter) Seq("filtered") else Seq.empty[String] } ++ { + if (useSmallPages) Seq("small pages") else Seq.empty[String] + } ++ { if (useSmallRowGroups) Seq("small row groups") else Seq.empty[String] } ++ { + if (useSmallSplits) Seq("small splits") else Seq.empty[String] + } ++ { if (useDataSourceV2) Seq("datasource v2") else Seq.empty[String] } + }.mkString(", ") + + def sqlConfs: Seq[(String, String)] = Seq( + // TODO: remove this change after customized parquet options as `block_size`, `page_size` + // been fully supported. + "spark.gluten.sql.native.writer.enabled" -> "false", + SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> useVectorizedReader.toString, + SQLConf.FILES_MAX_PARTITION_BYTES.key -> filesMaxPartitionBytes.toString + ) ++ { if (useDataSourceV2) Seq(SQLConf.USE_V1_SOURCE_LIST.key -> "") else Seq.empty } + } + + for (useVectorizedReader <- Seq(true, false)) + for (useDataSourceV2 <- Seq(true, false)) + for (useSmallRowGroups <- Seq(true, false)) + for (useSmallPages <- Seq(true, false)) + for (useFilter <- Seq(true, false)) + for (useSmallSplits <- Seq(useSmallRowGroups, false).distinct) { + val conf = GlutenRowIndexTestConf( + useVectorizedReader = useVectorizedReader, + useDataSourceV2 = useDataSourceV2, + useSmallRowGroups = useSmallRowGroups, + useSmallPages = useSmallPages, + useFilter = useFilter, + useSmallSplits = useSmallSplits + ) + testRowIndexGeneration("row index generation", conf) + } + + private def testRowIndexGeneration(label: String, conf: GlutenRowIndexTestConf): Unit = { + testGluten(s"$label - ${conf.desc}") { + withSQLConf(conf.sqlConfs: _*) { + withTempPath { + path => + val rowIndexColName = FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + val numRecordsPerFile = conf.numRows / conf.numFiles + val (skipCentileFirst, skipCentileMidLeft, skipCentileMidRight, skipCentileLast) = + (0.2, 0.4, 0.6, 0.8) + val expectedRowIdxCol = "expected_rowIdx_col" + val df = spark + .range(0, conf.numRows, 1, conf.numFiles) + .toDF("id") + .withColumn("dummy_col", ($"id" / 55).cast("int")) + .withColumn(expectedRowIdxCol, ($"id" % numRecordsPerFile).cast("int")) + + // With row index in schema. + val schemaWithRowIdx = df.schema.add(rowIndexColName, LongType, nullable = true) + + df.write + .format(conf.writeFormat) + .option(ParquetOutputFormat.BLOCK_SIZE, conf.rowGroupSize) + .option(ParquetOutputFormat.PAGE_SIZE, conf.pageSize) + .option(ParquetOutputFormat.DICTIONARY_PAGE_SIZE, conf.pageSize) + .save(path.getAbsolutePath) + val dfRead = spark.read + .format(conf.readFormat) + .schema(schemaWithRowIdx) + .load(path.getAbsolutePath) + + // Verify that the produced files are laid out as expected. + if (conf.useSmallRowGroups) { + if (conf.useSmallPages) { + assertIntermediateRowGroups(path) + } else { + assertTinyRowGroups(path) + } + } else { + assertOneRowGroup(path) + } + + val dfToAssert = if (conf.useFilter) { + // Add a filter such that we skip 60% of the records: + // [0%, 20%], [40%, 60%], [80%, 100%] + dfRead.filter( + ($"id" >= (skipCentileFirst * conf.numRows).toInt && + $"id" < (skipCentileMidLeft * conf.numRows).toInt) || + ($"id" >= (skipCentileMidRight * conf.numRows).toInt && + $"id" < (skipCentileLast * conf.numRows).toInt)) + } else { + dfRead + } + + var numPartitions: Long = 0 + var numOutputRows: Long = 0 + dfToAssert.collect() + logInfo(dfToAssert.queryExecution.executedPlan.toString()) + dfToAssert.queryExecution.executedPlan.foreach { + case a: BatchScanExec => + numPartitions += a.inputRDD.partitions.length + numOutputRows += a.metrics("numOutputRows").value + case b: FileSourceScanExec => + numPartitions += b.inputRDD.partitions.length + numOutputRows += b.metrics("numOutputRows").value + case c: BatchScanExecTransformer => + numPartitions += c.inputRDD.partitions.length + numOutputRows += c.metrics("numOutputRows").value + case f: FileSourceScanExecTransformer => + numPartitions += f.inputRDD.partitions.length + numOutputRows += f.metrics("numOutputRows").value + case _ => + } + assert(numPartitions > 0) + assert(numOutputRows > 0) + + if (conf.useSmallSplits) { + // SPARK-39634: Until the fix the fix for PARQUET-2161 is available is available, + // it is not possible to split Parquet files into multiple partitions while generating + // row indexes. + // assert(numPartitions >= 2 * conf.numFiles) + } + + // Assert that every rowIdx value matches the value in `expectedRowIdx`. + assert( + dfToAssert + .filter(s"$rowIndexColName != $expectedRowIdxCol") + .count() == 0) + + if (conf.useFilter) { + if (conf.useSmallRowGroups) { + assert(numOutputRows < conf.numRows) + } + + val minMaxRowIndexes = + dfToAssert.select(max(col(rowIndexColName)), min(col(rowIndexColName))).collect() + val (expectedMaxRowIdx, expectedMinRowIdx) = if (conf.numFiles == 1) { + // When there is a single file, we still have row group skipping, + // but that should not affect the produced rowIdx. + (conf.numRows * skipCentileLast - 1, conf.numRows * skipCentileFirst) + } else { + // For simplicity, the chosen filter skips the whole files. + // Thus all unskipped files will have the same max and min rowIdx values. + (numRecordsPerFile - 1, 0) + } + assert(minMaxRowIndexes(0).get(0) == expectedMaxRowIdx) + assert(minMaxRowIndexes(0).get(1) == expectedMinRowIdx) + if (!conf.useMultipleFiles) { + val skippedValues = List.range(0, (skipCentileFirst * conf.numRows).toInt) ++ + List.range( + (skipCentileMidLeft * conf.numRows).toInt, + (skipCentileMidRight * conf.numRows).toInt) ++ + List.range((skipCentileLast * conf.numRows).toInt, conf.numRows) + // rowIdx column should not have any of the `skippedValues`. + assert( + dfToAssert + .filter(col(rowIndexColName).isin(skippedValues: _*)) + .count() == 0) + } + } else { + assert(numOutputRows == conf.numRows) + // When there is no filter, the rowIdx values should be in range + // [0-`numRecordsPerFile`]. + val expectedRowIdxValues = List.range(0, numRecordsPerFile) + assert( + dfToAssert + .filter(col(rowIndexColName).isin(expectedRowIdxValues: _*)) + .count() == conf.numRows) + } + } + } + } + } + + for (useDataSourceV2 <- Seq(true, false)) { + val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2) + + testGluten(s"invalid row index column type - ${conf.desc}") { + withSQLConf(conf.sqlConfs: _*) { + withTempPath { + path => + val df = spark.range(0, 10, 1, 1).toDF("id") + val schemaWithRowIdx = df.schema + .add(FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME, StringType) + + df.write + .format(conf.writeFormat) + .save(path.getAbsolutePath) + + val dfRead = spark.read + .format(conf.readFormat) + .schema(schemaWithRowIdx) + .load(path.getAbsolutePath) -class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait {} + val exception = intercept[Exception](dfRead.collect()) + assert(exception.getMessage.contains(FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME)) + } + } + } + } +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index a5981941146e..61353d99f7d1 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1198,8 +1198,9 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenResolveDefaultColumnsSuite] enableSuite[GlutenSubqueryHintPropagationSuite] enableSuite[GlutenUrlFunctionsSuite] - // Row index metadata column support in Velox isn't ready yet, refer velox-9147 - // enableSuite[GlutenParquetRowIndexSuite] + enableSuite[GlutenParquetRowIndexSuite] + .excludeByPrefix("row index generation") + .excludeByPrefix("invalid row index column type") enableSuite[GlutenBitmapExpressionsQuerySuite] enableSuite[GlutenEmptyInSuite] enableSuite[GlutenRuntimeNullChecksV2Writes] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala index acf6a2b6384d..abf21651f827 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -16,6 +16,346 @@ */ package org.apache.spark.sql.execution.datasources.parquet +import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer} + import org.apache.spark.sql.GlutenSQLTestsBaseTrait +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetDataSourceV2 +import org.apache.spark.sql.functions.{col, max, min} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{LongType, StringType} + +import org.apache.hadoop.fs.Path +import org.apache.parquet.column.ParquetProperties._ +import org.apache.parquet.format.converter.ParquetMetadataConverter +import org.apache.parquet.hadoop.ParquetOutputFormat +import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE + +import java.io.File + +import scala.collection.JavaConverters._ + +class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait { + import testImplicits._ + + private def readRowGroupRowCounts(path: String): Seq[Long] = { + ParquetFooterReader + .readFooter( + spark.sessionState.newHadoopConf(), + new Path(path), + ParquetMetadataConverter.NO_FILTER) + .getBlocks + .asScala + .map(_.getRowCount) + } + + private def readRowGroupRowCounts(dir: File): Seq[Seq[Long]] = { + assert(dir.isDirectory) + dir + .listFiles() + .filter(f => f.isFile && f.getName.endsWith("parquet")) + .map(f => readRowGroupRowCounts(f.getAbsolutePath)) + .toSeq + } + + /** Do the files contain exactly one row group? */ + private def assertOneRowGroup(dir: File): Unit = { + readRowGroupRowCounts(dir).foreach { + rcs => assert(rcs.length == 1, "expected one row group per file") + } + } + + /** + * Do the files have a good layout to test row group skipping (both range metadata filter, and by + * using min/max). + */ + private def assertTinyRowGroups(dir: File): Unit = { + readRowGroupRowCounts(dir).foreach { + rcs => + assert(rcs.length > 1, "expected multiple row groups per file") + assert(rcs.last <= DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK) + assert( + rcs.reverse.tail.distinct == Seq(DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK), + "expected row groups with minimal row count") + } + } + + /** + * Do the files have a good layout to test a combination of page skipping and row group skipping? + */ + private def assertIntermediateRowGroups(dir: File): Unit = { + readRowGroupRowCounts(dir).foreach { + rcs => + assert(rcs.length >= 3, "expected at least 3 row groups per file") + rcs.reverse.tail.foreach { + rc => + assert( + rc > DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK, + "expected row groups larger than minimal row count") + } + } + } + + case class GlutenRowIndexTestConf( + numRows: Long = 10000L, + useMultipleFiles: Boolean = false, + useVectorizedReader: Boolean = true, + useSmallPages: Boolean = false, + useSmallRowGroups: Boolean = false, + useSmallSplits: Boolean = false, + useFilter: Boolean = false, + useDataSourceV2: Boolean = false) { + + val NUM_MULTIPLE_FILES = 4 + // The test doesn't work correctly if the number of records per file is uneven. + assert(!useMultipleFiles || (numRows % NUM_MULTIPLE_FILES == 0)) + + def numFiles: Int = if (useMultipleFiles) { NUM_MULTIPLE_FILES } + else { 1 } + + def rowGroupSize: Long = if (useSmallRowGroups) { + if (useSmallPages) { + // Each file will contain multiple row groups. All of them (except for the last one) + // will contain more than DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK, so that individual + // pages within the row group can be skipped. + 2048L + } else { + // Each file will contain multiple row groups. All of them (except for the last one) + // will contain exactly DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK records. + 64L + } + } else { + // Each file will contain a single row group. + DEFAULT_BLOCK_SIZE + } + + def pageSize: Long = if (useSmallPages) { + // Each page (except for the last one for each column) will contain exactly + // DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK records. + 64L + } else { + DEFAULT_PAGE_SIZE + } + + def writeFormat: String = "parquet" + def readFormat: String = if (useDataSourceV2) { + classOf[ParquetDataSourceV2].getCanonicalName + } else { + "parquet" + } + + assert(useSmallRowGroups || !useSmallSplits) + def filesMaxPartitionBytes: Long = if (useSmallSplits) { + 256L + } else { + SQLConf.FILES_MAX_PARTITION_BYTES.defaultValue.get + } + + def desc: String = { + { if (useVectorizedReader) Seq("vectorized reader") else Seq("parquet-mr reader") } ++ { + if (useMultipleFiles) Seq("many files") else Seq.empty[String] + } ++ { if (useFilter) Seq("filtered") else Seq.empty[String] } ++ { + if (useSmallPages) Seq("small pages") else Seq.empty[String] + } ++ { if (useSmallRowGroups) Seq("small row groups") else Seq.empty[String] } ++ { + if (useSmallSplits) Seq("small splits") else Seq.empty[String] + } ++ { if (useDataSourceV2) Seq("datasource v2") else Seq.empty[String] } + }.mkString(", ") + + def sqlConfs: Seq[(String, String)] = Seq( + // TODO: remove this change after customized parquet options as `block_size`, `page_size` + // been fully supported. + "spark.gluten.sql.native.writer.enabled" -> "false", + SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> useVectorizedReader.toString, + SQLConf.FILES_MAX_PARTITION_BYTES.key -> filesMaxPartitionBytes.toString + ) ++ { if (useDataSourceV2) Seq(SQLConf.USE_V1_SOURCE_LIST.key -> "") else Seq.empty } + } + + for (useVectorizedReader <- Seq(true, false)) + for (useDataSourceV2 <- Seq(true, false)) + for (useSmallRowGroups <- Seq(true, false)) + for (useSmallPages <- Seq(true, false)) + for (useFilter <- Seq(true, false)) + for (useSmallSplits <- Seq(useSmallRowGroups, false).distinct) { + val conf = GlutenRowIndexTestConf( + useVectorizedReader = useVectorizedReader, + useDataSourceV2 = useDataSourceV2, + useSmallRowGroups = useSmallRowGroups, + useSmallPages = useSmallPages, + useFilter = useFilter, + useSmallSplits = useSmallSplits + ) + testRowIndexGeneration("row index generation", conf) + } + + private def testRowIndexGeneration(label: String, conf: GlutenRowIndexTestConf): Unit = { + testGluten(s"$label - ${conf.desc}") { + withSQLConf(conf.sqlConfs: _*) { + withTempPath { + path => + // Read row index using _metadata.row_index if that is supported by the file format. + val rowIndexMetadataColumnSupported = conf.readFormat match { + case "parquet" => true + case _ => false + } + val rowIndexColName = if (rowIndexMetadataColumnSupported) { + s"${FileFormat.METADATA_NAME}.${ParquetFileFormat.ROW_INDEX}" + } else { + ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + } + val numRecordsPerFile = conf.numRows / conf.numFiles + val (skipCentileFirst, skipCentileMidLeft, skipCentileMidRight, skipCentileLast) = + (0.2, 0.4, 0.6, 0.8) + val expectedRowIdxCol = "expected_rowIdx_col" + val df = spark + .range(0, conf.numRows, 1, conf.numFiles) + .toDF("id") + .withColumn("dummy_col", ($"id" / 55).cast("int")) + .withColumn(expectedRowIdxCol, ($"id" % numRecordsPerFile).cast("int")) + + // Add row index to schema if required. + val schemaWithRowIdx = if (rowIndexMetadataColumnSupported) { + df.schema + } else { + df.schema.add(rowIndexColName, LongType, nullable = true) + } + + logInfo(s"gyytest schemaWithRowIndex $schemaWithRowIdx") + + df.write + .format(conf.writeFormat) + .option(ParquetOutputFormat.BLOCK_SIZE, conf.rowGroupSize) + .option(ParquetOutputFormat.PAGE_SIZE, conf.pageSize) + .option(ParquetOutputFormat.DICTIONARY_PAGE_SIZE, conf.pageSize) + .save(path.getAbsolutePath) + val dfRead = spark.read + .format(conf.readFormat) + .schema(schemaWithRowIdx) + .load(path.getAbsolutePath) + // Verify that the produced files are laid out as expected. + if (conf.useSmallRowGroups) { + if (conf.useSmallPages) { + assertIntermediateRowGroups(path) + } else { + assertTinyRowGroups(path) + } + } else { + assertOneRowGroup(path) + } + + val dfToAssert = if (conf.useFilter) { + // Add a filter such that we skip 60% of the records: + // [0%, 20%], [40%, 60%], [80%, 100%] + dfRead.filter( + ($"id" >= (skipCentileFirst * conf.numRows).toInt && + $"id" < (skipCentileMidLeft * conf.numRows).toInt) || + ($"id" >= (skipCentileMidRight * conf.numRows).toInt && + $"id" < (skipCentileLast * conf.numRows).toInt)) + } else { + dfRead + } + + var numPartitions: Long = 0 + var numOutputRows: Long = 0 + dfToAssert.collect() + logInfo(dfToAssert.queryExecution.executedPlan.toString()) + dfToAssert.queryExecution.executedPlan.foreach { + case a: BatchScanExec => + numPartitions += a.inputRDD.partitions.length + numOutputRows += a.metrics("numOutputRows").value + case b: FileSourceScanExec => + numPartitions += b.inputRDD.partitions.length + numOutputRows += b.metrics("numOutputRows").value + case c: BatchScanExecTransformer => + numPartitions += c.inputRDD.partitions.length + numOutputRows += c.metrics("numOutputRows").value + case f: FileSourceScanExecTransformer => + numPartitions += f.inputRDD.partitions.length + numOutputRows += f.metrics("numOutputRows").value + case _ => + } + assert(numPartitions > 0) + assert(numOutputRows > 0) + + if (conf.useSmallSplits) { + assert(numPartitions >= 2 * conf.numFiles) + } + + // Assert that every rowIdx value matches the value in `expectedRowIdx`. + assert( + dfToAssert + .filter(s"$rowIndexColName != $expectedRowIdxCol") + .count() == 0) + + if (conf.useFilter) { + if (conf.useSmallRowGroups) { + assert(numOutputRows < conf.numRows) + } + + val minMaxRowIndexes = + dfToAssert.select(max(col(rowIndexColName)), min(col(rowIndexColName))).collect() + val (expectedMaxRowIdx, expectedMinRowIdx) = if (conf.numFiles == 1) { + // When there is a single file, we still have row group skipping, + // but that should not affect the produced rowIdx. + (conf.numRows * skipCentileLast - 1, conf.numRows * skipCentileFirst) + } else { + // For simplicity, the chosen filter skips the whole files. + // Thus all unskipped files will have the same max and min rowIdx values. + (numRecordsPerFile - 1, 0) + } + assert(minMaxRowIndexes(0).get(0) == expectedMaxRowIdx) + assert(minMaxRowIndexes(0).get(1) == expectedMinRowIdx) + if (!conf.useMultipleFiles) { + val skippedValues = List.range(0, (skipCentileFirst * conf.numRows).toInt) ++ + List.range( + (skipCentileMidLeft * conf.numRows).toInt, + (skipCentileMidRight * conf.numRows).toInt) ++ + List.range((skipCentileLast * conf.numRows).toInt, conf.numRows) + // rowIdx column should not have any of the `skippedValues`. + assert( + dfToAssert + .filter(col(rowIndexColName).isin(skippedValues: _*)) + .count() == 0) + } + } else { + // assert(numOutputRows == conf.numRows) + // When there is no filter, the rowIdx values should be in range + // [0-`numRecordsPerFile`]. + val expectedRowIdxValues = List.range(0, numRecordsPerFile) + assert( + dfToAssert + .filter(col(rowIndexColName).isin(expectedRowIdxValues: _*)) + .count() == conf.numRows) + } + } + } + } + } + for (useDataSourceV2 <- Seq(true, false)) { + val conf = GlutenRowIndexTestConf(useDataSourceV2 = useDataSourceV2) + + testGluten(s"invalid row index column type - ${conf.desc}") { + withSQLConf(conf.sqlConfs: _*) { + withTempPath { + path => + val df = spark.range(0, 10, 1, 1).toDF("id") + val schemaWithRowIdx = df.schema + .add(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME, StringType) + + df.write + .format(conf.writeFormat) + .save(path.getAbsolutePath) + + val dfRead = spark.read + .format(conf.readFormat) + .schema(schemaWithRowIdx) + .load(path.getAbsolutePath) -class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait {} + val exception = intercept[Exception](dfRead.collect()) + assert(exception.getMessage.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME)) + } + } + } + } +} diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 007381fe6158..4db784782c1e 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -42,6 +42,13 @@ abstract class BatchScanExecShim( // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics: Map[String, SQLMetric] = Map() + def metadataColumns: Seq[AttributeReference] = Seq.empty + + def hasUnsupportedColumns: Boolean = { + // Below name has special meaning in Velox. + output.exists(a => a.name == "$path" || a.name == "$bucket") + } + override def doExecuteColumnar(): RDD[ColumnarBatch] = { throw new UnsupportedOperationException("Need to implement this method") } diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index dcfb5c9501f3..76556052c758 100644 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -46,6 +46,20 @@ abstract class BatchScanExecShim( // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics: Map[String, SQLMetric] = Map() + lazy val metadataColumns: Seq[AttributeReference] = output.collect { + case FileSourceMetadataAttribute(attr) => attr + } + + def hasUnsupportedColumns: Boolean = { + // TODO, fallback if user define same name column due to we can't right now + // detect which column is metadata column which is user defined column. + val metadataColumnsNames = metadataColumns.map(_.name) + output + .filterNot(metadataColumns.toSet) + .exists(v => metadataColumnsNames.contains(v.name)) || + output.exists(a => a.name == "$path" || a.name == "$bucket") + } + override def doExecuteColumnar(): RDD[ColumnarBatch] = { throw new UnsupportedOperationException("Need to implement this method") } diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 4fc09f3aef35..15455d51c7a9 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -66,7 +66,10 @@ abstract class FileSourceScanExecShim( .filterNot(metadataColumns.toSet) .exists(v => metadataColumnsNames.contains(v.name)) || // Below name has special meaning in Velox. - output.exists(a => a.name == "$path" || a.name == "$bucket") + output.exists( + a => + a.name == "$path" || a.name == "$bucket" || + a.name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) } def isMetadataColumn(attr: Attribute): Boolean = metadataColumns.contains(attr) diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 4c12356d6378..ca9a7eb2d071 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.aggregate.Aggregation import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} import org.apache.spark.sql.connector.read.SupportsRuntimeV2Filtering +import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan import org.apache.spark.sql.execution.metric.SQLMetric @@ -55,6 +56,25 @@ abstract class BatchScanExecShim( // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics: Map[String, SQLMetric] = Map() + lazy val metadataColumns: Seq[AttributeReference] = output.collect { + case FileSourceConstantMetadataAttribute(attr) => attr + case FileSourceGeneratedMetadataAttribute(attr) => attr + } + + def hasUnsupportedColumns: Boolean = { + // TODO, fallback if user define same name column due to we can't right now + // detect which column is metadata column which is user defined column. + val metadataColumnsNames = metadataColumns.map(_.name) + metadataColumnsNames.contains(FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) || + output + .filterNot(metadataColumns.toSet) + .exists(v => metadataColumnsNames.contains(v.name)) || + output.exists( + a => + a.name == "$path" || a.name == "$bucket" || + a.name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) + } + override def doExecuteColumnar(): RDD[ColumnarBatch] = { throw new UnsupportedOperationException("Need to implement this method") } diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index b3bb2d2934e3..6295bcbc46d4 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -71,7 +71,10 @@ abstract class FileSourceScanExecShim( output .filterNot(metadataColumns.toSet) .exists(v => metadataColumnsNames.contains(v.name)) || - output.exists(a => a.name == "$path" || a.name == "$bucket") + output.exists( + a => + a.name == "$path" || a.name == "$bucket" || + a.name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) } def isMetadataColumn(attr: Attribute): Boolean = metadataColumns.contains(attr) diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index bb380609792a..47adf16fb0e7 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.aggregate.Aggregation import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} import org.apache.spark.sql.connector.read.SupportsRuntimeV2Filtering +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan import org.apache.spark.sql.execution.metric.SQLMetric @@ -57,6 +58,25 @@ abstract class BatchScanExecShim( // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics: Map[String, SQLMetric] = Map() + lazy val metadataColumns: Seq[AttributeReference] = output.collect { + case FileSourceConstantMetadataAttribute(attr) => attr + case FileSourceGeneratedMetadataAttribute(attr, _) => attr + } + + def hasUnsupportedColumns: Boolean = { + // TODO, fallback if user define same name column due to we can't right now + // detect which column is metadata column which is user defined column. + val metadataColumnsNames = metadataColumns.map(_.name) + metadataColumnsNames.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) || + output + .filterNot(metadataColumns.toSet) + .exists(v => metadataColumnsNames.contains(v.name)) || + output.exists( + a => + a.name == "$path" || a.name == "$bucket" || + a.name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) + } + override def doExecuteColumnar(): RDD[ColumnarBatch] = { throw new UnsupportedOperationException("Need to implement this method") } From 2984bd740e9bfc02d496a5eb39916461642073c2 Mon Sep 17 00:00:00 2001 From: Yibing <568420827@qq.com> Date: Wed, 15 May 2024 16:30:28 +0800 Subject: [PATCH 078/402] [GLUTEN-5731][CORE] Fix the logic to calculate shuffle write time in RssPartitionWriter (#5742) Co-authored-by: yibing --- cpp/core/shuffle/rss/RssPartitionWriter.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/core/shuffle/rss/RssPartitionWriter.cc b/cpp/core/shuffle/rss/RssPartitionWriter.cc index 72bcf1d3a4b0..15981bf8ded8 100644 --- a/cpp/core/shuffle/rss/RssPartitionWriter.cc +++ b/cpp/core/shuffle/rss/RssPartitionWriter.cc @@ -56,8 +56,6 @@ arrow::Status RssPartitionWriter::evict( bool reuseBuffers, bool hasComplexType) { rawPartitionLengths_[partitionId] += inMemoryPayload->getBufferSize(); - - ScopedTimer timer(&spillTime_); auto payloadType = (codec_ && inMemoryPayload->numRows() >= options_.compressionThreshold) ? Payload::Type::kCompressed : Payload::Type::kUncompressed; @@ -69,6 +67,7 @@ arrow::Status RssPartitionWriter::evict( payload = nullptr; // Invalidate payload immediately. // Push. + ScopedTimer timer(&spillTime_); ARROW_ASSIGN_OR_RAISE(auto buffer, rssBufferOs->Finish()); bytesEvicted_[partitionId] += rssClient_->pushPartitionData( partitionId, reinterpret_cast(const_cast(buffer->data())), buffer->size()); From 9d2a13bffb4292f17714bfbba96638aeadb91062 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Thu, 16 May 2024 11:06:18 +0800 Subject: [PATCH 079/402] [VL] Move velox related configs to VeloxConfig.h (#5743) --- cpp/core/config/GlutenConfig.h | 7 - cpp/velox/compute/VeloxBackend.cc | 182 ++++++------------ cpp/velox/compute/VeloxBackend.h | 12 +- cpp/velox/compute/VeloxRuntime.cc | 4 +- cpp/velox/compute/WholeStageResultIterator.cc | 47 +---- cpp/velox/config/VeloxConfig.h | 127 ++++++++++++ 6 files changed, 192 insertions(+), 187 deletions(-) create mode 100644 cpp/velox/config/VeloxConfig.h diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index 3c47fb5479bd..16a18f6be903 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -61,13 +61,6 @@ const std::string kShuffleCompressionCodecBackend = "spark.gluten.sql.columnar.s const std::string kQatBackendName = "qat"; const std::string kIaaBackendName = "iaa"; -// Velox conf -const std::string kGlogVerboseLevel = "spark.gluten.sql.columnar.backend.velox.glogVerboseLevel"; -const uint32_t kGlogVerboseLevelDefault = 0; -const uint32_t kGlogVerboseLevelMaximum = 99; -const std::string kGlogSeverityLevel = "spark.gluten.sql.columnar.backend.velox.glogSeverityLevel"; -const uint32_t kGlogSeverityLevelDefault = 1; - std::unordered_map parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength); diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 044c8aa0ea7e..b2fb1c964e22 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -24,8 +24,6 @@ #include "operators/plannodes/RowVectorStream.h" #include "utils/ConfigExtractor.h" -#include "shuffle/VeloxShuffleReader.h" - #ifdef GLUTEN_ENABLE_QAT #include "utils/qat/QatCodec.h" #endif @@ -33,7 +31,7 @@ #include "utils/qpl/qpl_codec.h" #endif #include "compute/VeloxRuntime.h" -#include "config/GlutenConfig.h" +#include "config/VeloxConfig.h" #include "jni/JniFileSystem.h" #include "operators/functions/SparkTokenizer.h" #include "udf/UdfLoader.h" @@ -54,71 +52,6 @@ DEFINE_int32(gluten_velox_aysnc_timeout_on_task_stopping, 30000, "Aysnc timout w using namespace facebook; -namespace { - -const std::string kEnableUserExceptionStacktrace = - "spark.gluten.sql.columnar.backend.velox.enableUserExceptionStacktrace"; -const bool kEnableUserExceptionStacktraceDefault = true; - -const std::string kEnableSystemExceptionStacktrace = - "spark.gluten.sql.columnar.backend.velox.enableSystemExceptionStacktrace"; -const bool kEnableSystemExceptionStacktraceDefault = true; - -const std::string kMemoryUseHugePages = "spark.gluten.sql.columnar.backend.velox.memoryUseHugePages"; -const bool kMemoryUseHugePagesDefault = false; - -const std::string kHiveConnectorId = "test-hive"; -const std::string kVeloxCacheEnabled = "spark.gluten.sql.columnar.backend.velox.cacheEnabled"; - -// memory cache -const std::string kVeloxMemCacheSize = "spark.gluten.sql.columnar.backend.velox.memCacheSize"; -const uint64_t kVeloxMemCacheSizeDefault = 1073741824; // 1G - -// ssd cache -const std::string kVeloxSsdCacheSize = "spark.gluten.sql.columnar.backend.velox.ssdCacheSize"; -const uint64_t kVeloxSsdCacheSizeDefault = 1073741824; // 1G -const std::string kVeloxSsdCachePath = "spark.gluten.sql.columnar.backend.velox.ssdCachePath"; -const std::string kVeloxSsdCachePathDefault = "/tmp/"; -const std::string kVeloxSsdCacheShards = "spark.gluten.sql.columnar.backend.velox.ssdCacheShards"; -const uint32_t kVeloxSsdCacheShardsDefault = 1; -const std::string kVeloxSsdCacheIOThreads = "spark.gluten.sql.columnar.backend.velox.ssdCacheIOThreads"; -const uint32_t kVeloxSsdCacheIOThreadsDefault = 1; -const std::string kVeloxSsdODirectEnabled = "spark.gluten.sql.columnar.backend.velox.ssdODirect"; - -// async -const std::string kVeloxIOThreads = "spark.gluten.sql.columnar.backend.velox.IOThreads"; -const uint32_t kVeloxIOThreadsDefault = 0; -const std::string kVeloxAsyncTimeoutOnTaskStopping = - "spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping"; -const int32_t kVeloxAsyncTimeoutOnTaskStoppingDefault = 30000; // 30s - -// udf -const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.udfLibraryPaths"; - -// spill -const std::string kMaxSpillFileSize = "spark.gluten.sql.columnar.backend.velox.maxSpillFileSize"; -const uint64_t kMaxSpillFileSizeDefault = 1L * 1024 * 1024 * 1024; - -// backtrace allocation -const std::string kBacktraceAllocation = "spark.gluten.backtrace.allocation"; - -// VeloxShuffleReader print flag. -const std::string kVeloxShuffleReaderPrintFlag = "spark.gluten.velox.shuffleReaderPrintFlag"; - -const std::string kVeloxFileHandleCacheEnabled = "spark.gluten.sql.columnar.backend.velox.fileHandleCacheEnabled"; -const bool kVeloxFileHandleCacheEnabledDefault = false; - -/* configs for file read in velox*/ -const std::string kDirectorySizeGuess = "spark.gluten.sql.columnar.backend.velox.directorySizeGuess"; -const std::string kFilePreloadThreshold = "spark.gluten.sql.columnar.backend.velox.filePreloadThreshold"; -const std::string kPrefetchRowGroups = "spark.gluten.sql.columnar.backend.velox.prefetchRowGroups"; -const std::string kLoadQuantum = "spark.gluten.sql.columnar.backend.velox.loadQuantum"; -const std::string kMaxCoalescedDistanceBytes = "spark.gluten.sql.columnar.backend.velox.maxCoalescedDistanceBytes"; -const std::string kMaxCoalescedBytes = "spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes"; -const std::string kCachePrefetchMinPct = "spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct"; - -} // namespace - namespace gluten { namespace { @@ -128,25 +61,22 @@ gluten::Runtime* veloxRuntimeFactory(const std::unordered_map& conf) { - backendConf_ = conf; + backendConf_ = std::make_shared(conf); // Register Velox runtime factory gluten::Runtime::registerFactory(gluten::kVeloxRuntimeKind, veloxRuntimeFactory); - std::shared_ptr veloxcfg = - std::make_shared(conf); - - if (veloxcfg->get(kDebugModeEnabled, false)) { - LOG(INFO) << "VeloxBackend config:" << printConfig(veloxcfg->valuesCopy()); + if (backendConf_->get(kDebugModeEnabled, false)) { + LOG(INFO) << "VeloxBackend config:" << printConfig(backendConf_->valuesCopy()); } // Init glog and log level. - if (!veloxcfg->get(kDebugModeEnabled, false)) { - FLAGS_v = veloxcfg->get(kGlogVerboseLevel, kGlogVerboseLevelDefault); - FLAGS_minloglevel = veloxcfg->get(kGlogSeverityLevel, kGlogSeverityLevelDefault); + if (!backendConf_->get(kDebugModeEnabled, false)) { + FLAGS_v = backendConf_->get(kGlogVerboseLevel, kGlogVerboseLevelDefault); + FLAGS_minloglevel = backendConf_->get(kGlogSeverityLevel, kGlogSeverityLevelDefault); } else { - if (veloxcfg->isValueExists(kGlogVerboseLevel)) { - FLAGS_v = veloxcfg->get(kGlogVerboseLevel, kGlogVerboseLevelDefault); + if (backendConf_->isValueExists(kGlogVerboseLevel)) { + FLAGS_v = backendConf_->get(kGlogVerboseLevel, kGlogVerboseLevelDefault); } else { FLAGS_v = kGlogVerboseLevelMaximum; } @@ -159,27 +89,27 @@ void VeloxBackend::init(const std::unordered_map& conf // Set velox_exception_user_stacktrace_enabled. FLAGS_velox_exception_user_stacktrace_enabled = - veloxcfg->get(kEnableUserExceptionStacktrace, kEnableUserExceptionStacktraceDefault); + backendConf_->get(kEnableUserExceptionStacktrace, kEnableUserExceptionStacktraceDefault); // Set velox_exception_system_stacktrace_enabled. FLAGS_velox_exception_system_stacktrace_enabled = - veloxcfg->get(kEnableSystemExceptionStacktrace, kEnableSystemExceptionStacktraceDefault); + backendConf_->get(kEnableSystemExceptionStacktrace, kEnableSystemExceptionStacktraceDefault); // Set velox_memory_use_hugepages. - FLAGS_velox_memory_use_hugepages = veloxcfg->get(kMemoryUseHugePages, kMemoryUseHugePagesDefault); + FLAGS_velox_memory_use_hugepages = backendConf_->get(kMemoryUseHugePages, kMemoryUseHugePagesDefault); // Async timeout. FLAGS_gluten_velox_aysnc_timeout_on_task_stopping = - veloxcfg->get(kVeloxAsyncTimeoutOnTaskStopping, kVeloxAsyncTimeoutOnTaskStoppingDefault); + backendConf_->get(kVeloxAsyncTimeoutOnTaskStopping, kVeloxAsyncTimeoutOnTaskStoppingDefault); // Set backtrace_allocation - gluten::backtrace_allocation = veloxcfg->get(kBacktraceAllocation, false); + gluten::backtrace_allocation = backendConf_->get(kBacktraceAllocation, false); // Setup and register. velox::filesystems::registerLocalFileSystem(); - initJolFilesystem(veloxcfg); - initCache(veloxcfg); - initConnector(veloxcfg); + initJolFilesystem(); + initCache(); + initConnector(); // Register Velox functions registerAllFunctions(); @@ -189,7 +119,7 @@ void VeloxBackend::init(const std::unordered_map& conf } velox::exec::Operator::registerOperator(std::make_unique()); - initUdf(veloxcfg); + initUdf(); registerSparkTokenizer(); // initialize the global memory manager for current process @@ -201,8 +131,8 @@ facebook::velox::cache::AsyncDataCache* VeloxBackend::getAsyncDataCache() const } // JNI-or-local filesystem, for spilling-to-heap if we have extra JVM heap spaces -void VeloxBackend::initJolFilesystem(const std::shared_ptr& conf) { - int64_t maxSpillFileSize = conf->get(kMaxSpillFileSize, kMaxSpillFileSizeDefault); +void VeloxBackend::initJolFilesystem() { + int64_t maxSpillFileSize = backendConf_->get(kMaxSpillFileSize, kMaxSpillFileSizeDefault); // FIXME It's known that if spill compression is disabled, the actual spill file size may // in crease beyond this limit a little (maximum 64 rows which is by default @@ -210,18 +140,17 @@ void VeloxBackend::initJolFilesystem(const std::shared_ptr& conf) { - bool veloxCacheEnabled = conf->get(kVeloxCacheEnabled, false); - if (veloxCacheEnabled) { +void VeloxBackend::initCache() { + if (backendConf_->get(kVeloxCacheEnabled, false)) { FLAGS_ssd_odirect = true; - FLAGS_ssd_odirect = conf->get(kVeloxSsdODirectEnabled, false); + FLAGS_ssd_odirect = backendConf_->get(kVeloxSsdODirectEnabled, false); - uint64_t memCacheSize = conf->get(kVeloxMemCacheSize, kVeloxMemCacheSizeDefault); - uint64_t ssdCacheSize = conf->get(kVeloxSsdCacheSize, kVeloxSsdCacheSizeDefault); - int32_t ssdCacheShards = conf->get(kVeloxSsdCacheShards, kVeloxSsdCacheShardsDefault); - int32_t ssdCacheIOThreads = conf->get(kVeloxSsdCacheIOThreads, kVeloxSsdCacheIOThreadsDefault); - std::string ssdCachePathPrefix = conf->get(kVeloxSsdCachePath, kVeloxSsdCachePathDefault); + uint64_t memCacheSize = backendConf_->get(kVeloxMemCacheSize, kVeloxMemCacheSizeDefault); + uint64_t ssdCacheSize = backendConf_->get(kVeloxSsdCacheSize, kVeloxSsdCacheSizeDefault); + int32_t ssdCacheShards = backendConf_->get(kVeloxSsdCacheShards, kVeloxSsdCacheShardsDefault); + int32_t ssdCacheIOThreads = backendConf_->get(kVeloxSsdCacheIOThreads, kVeloxSsdCacheIOThreadsDefault); + std::string ssdCachePathPrefix = backendConf_->get(kVeloxSsdCachePath, kVeloxSsdCachePathDefault); cachePathPrefix_ = ssdCachePathPrefix; cacheFilePrefix_ = getCacheFilePrefix(); @@ -257,63 +186,64 @@ void VeloxBackend::initCache(const std::shared_ptr& conf) { +void VeloxBackend::initConnector() { // The configs below are used at process level. - auto mutableConf = std::make_shared(conf->valuesCopy()); + auto connectorConf = std::make_shared(backendConf_->valuesCopy()); - auto hiveConf = getHiveConfig(conf); + auto hiveConf = getHiveConfig(backendConf_); for (auto& [k, v] : hiveConf->valuesCopy()) { - mutableConf->setValue(k, v); + connectorConf->setValue(k, v); } #ifdef ENABLE_ABFS - const auto& confValue = conf->valuesCopy(); + const auto& confValue = backendConf_->valuesCopy(); for (auto& [k, v] : confValue) { if (k.find("fs.azure.account.key") == 0) { - mutableConf->setValue(k, v); + connectorConf->setValue(k, v); } else if (k.find("spark.hadoop.fs.azure.account.key") == 0) { constexpr int32_t accountKeyPrefixLength = 13; - mutableConf->setValue(k.substr(accountKeyPrefixLength), v); + connectorConf->setValue(k.substr(accountKeyPrefixLength), v); } } #endif - mutableConf->setValue( + connectorConf->setValue( velox::connector::hive::HiveConfig::kEnableFileHandleCache, - conf->get(kVeloxFileHandleCacheEnabled, kVeloxFileHandleCacheEnabledDefault) ? "true" : "false"); + backendConf_->get(kVeloxFileHandleCacheEnabled, kVeloxFileHandleCacheEnabledDefault) ? "true" : "false"); - mutableConf->setValue( + connectorConf->setValue( velox::connector::hive::HiveConfig::kMaxCoalescedBytes, - conf->get(kMaxCoalescedBytes, "67108864")); // 64M - mutableConf->setValue( + backendConf_->get(kMaxCoalescedBytes, "67108864")); // 64M + connectorConf->setValue( velox::connector::hive::HiveConfig::kMaxCoalescedDistanceBytes, - conf->get(kMaxCoalescedDistanceBytes, "1048576")); // 1M - mutableConf->setValue( - velox::connector::hive::HiveConfig::kPrefetchRowGroups, conf->get(kPrefetchRowGroups, "1")); - mutableConf->setValue( - velox::connector::hive::HiveConfig::kLoadQuantum, conf->get(kLoadQuantum, "268435456")); // 256M - mutableConf->setValue( + backendConf_->get(kMaxCoalescedDistanceBytes, "1048576")); // 1M + connectorConf->setValue( + velox::connector::hive::HiveConfig::kPrefetchRowGroups, backendConf_->get(kPrefetchRowGroups, "1")); + connectorConf->setValue( + velox::connector::hive::HiveConfig::kLoadQuantum, + backendConf_->get(kLoadQuantum, "268435456")); // 256M + connectorConf->setValue( velox::connector::hive::HiveConfig::kFooterEstimatedSize, - conf->get(kDirectorySizeGuess, "32768")); // 32K - mutableConf->setValue( + backendConf_->get(kDirectorySizeGuess, "32768")); // 32K + connectorConf->setValue( velox::connector::hive::HiveConfig::kFilePreloadThreshold, - conf->get(kFilePreloadThreshold, "1048576")); // 1M + backendConf_->get(kFilePreloadThreshold, "1048576")); // 1M // set cache_prefetch_min_pct default as 0 to force all loads are prefetched in DirectBufferInput. - FLAGS_cache_prefetch_min_pct = conf->get(kCachePrefetchMinPct, 0); + FLAGS_cache_prefetch_min_pct = backendConf_->get(kCachePrefetchMinPct, 0); - auto ioThreads = conf->get(kVeloxIOThreads, kVeloxIOThreadsDefault); + auto ioThreads = backendConf_->get(kVeloxIOThreads, kVeloxIOThreadsDefault); if (ioThreads > 0) { ioExecutor_ = std::make_unique(ioThreads); } velox::connector::registerConnector(std::make_shared( kHiveConnectorId, - std::make_shared(mutableConf->valuesCopy()), + std::make_shared(connectorConf->valuesCopy()), ioExecutor_.get())); } -void VeloxBackend::initUdf(const std::shared_ptr& conf) { - auto got = conf->get(kVeloxUdfLibraryPaths, ""); +void VeloxBackend::initUdf() { + auto got = backendConf_->get(kVeloxUdfLibraryPaths, ""); if (!got.empty()) { auto udfLoader = gluten::UdfLoader::getInstance(); udfLoader->loadUdfLibraries(got); @@ -335,7 +265,7 @@ VeloxBackend* VeloxBackend::get() { return instance_.get(); } -const std::unordered_map& VeloxBackend::getBackendConf() const { +const std::shared_ptr VeloxBackend::getBackendConf() const { return backendConf_; } diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h index a601d715cfa7..891bdd2cc408 100644 --- a/cpp/velox/compute/VeloxBackend.h +++ b/cpp/velox/compute/VeloxBackend.h @@ -53,7 +53,7 @@ class VeloxBackend { facebook::velox::cache::AsyncDataCache* getAsyncDataCache() const; - const std::unordered_map& getBackendConf() const; + const std::shared_ptr getBackendConf() const; void tearDown() { // Destruct IOThreadPoolExecutor will join all threads. @@ -68,11 +68,11 @@ class VeloxBackend { } void init(const std::unordered_map& conf); - void initCache(const std::shared_ptr& conf); - void initConnector(const std::shared_ptr& conf); - void initUdf(const std::shared_ptr& conf); + void initCache(); + void initConnector(); + void initUdf(); - void initJolFilesystem(const std::shared_ptr& conf); + void initJolFilesystem(); std::string getCacheFilePrefix() { return "cache." + boost::lexical_cast(boost::uuids::random_generator()()) + "."; @@ -90,7 +90,7 @@ class VeloxBackend { std::string cachePathPrefix_; std::string cacheFilePrefix_; - std::unordered_map backendConf_{}; + std::shared_ptr backendConf_; }; } // namespace gluten diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index 8314d0bd271a..a3e8c159c9ee 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -26,7 +26,7 @@ #include "compute/ResultIterator.h" #include "compute/Runtime.h" #include "compute/VeloxPlanConverter.h" -#include "config/GlutenConfig.h" +#include "config/VeloxConfig.h" #include "operators/serializer/VeloxRowToColumnarConverter.h" #include "shuffle/VeloxShuffleReader.h" #include "shuffle/VeloxShuffleWriter.h" @@ -256,7 +256,7 @@ std::unique_ptr VeloxRuntime::createColumnarBatchSerial } void VeloxRuntime::dumpConf(const std::string& path) { - auto backendConf = VeloxBackend::get()->getBackendConf(); + auto backendConf = VeloxBackend::get()->getBackendConf()->valuesCopy(); auto allConf = backendConf; for (const auto& pair : confMap_) { diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 83749061c1b8..006b37588005 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -17,13 +17,11 @@ #include "WholeStageResultIterator.h" #include "VeloxBackend.h" #include "VeloxRuntime.h" -#include "config/GlutenConfig.h" +#include "config/VeloxConfig.h" #include "velox/connectors/hive/HiveConfig.h" #include "velox/connectors/hive/HiveConnectorSplit.h" #include "velox/exec/PlanNodeStats.h" -#include "utils/ConfigExtractor.h" - #ifdef ENABLE_HDFS #include "utils/HdfsUtils.h" #endif @@ -33,49 +31,6 @@ using namespace facebook; namespace gluten { namespace { -// Velox configs -const std::string kHiveConnectorId = "test-hive"; - -// memory -const std::string kSpillStrategy = "spark.gluten.sql.columnar.backend.velox.spillStrategy"; -const std::string kSpillStrategyDefaultValue = "auto"; -const std::string kSpillThreadNum = "spark.gluten.sql.columnar.backend.velox.spillThreadNum"; -const uint32_t kSpillThreadNumDefaultValue = 0; -const std::string kAggregationSpillEnabled = "spark.gluten.sql.columnar.backend.velox.aggregationSpillEnabled"; -const std::string kJoinSpillEnabled = "spark.gluten.sql.columnar.backend.velox.joinSpillEnabled"; -const std::string kOrderBySpillEnabled = "spark.gluten.sql.columnar.backend.velox.orderBySpillEnabled"; - -// spill config -// refer to -// https://github.com/facebookincubator/velox/blob/95f3e80e77d046c12fbc79dc529366be402e9c2b/velox/docs/configs.rst#spilling -const std::string kMaxSpillLevel = "spark.gluten.sql.columnar.backend.velox.maxSpillLevel"; -const std::string kMaxSpillFileSize = "spark.gluten.sql.columnar.backend.velox.maxSpillFileSize"; -const std::string kSpillStartPartitionBit = "spark.gluten.sql.columnar.backend.velox.spillStartPartitionBit"; -const std::string kSpillPartitionBits = "spark.gluten.sql.columnar.backend.velox.spillPartitionBits"; -const std::string kMaxSpillRunRows = "spark.gluten.sql.columnar.backend.velox.MaxSpillRunRows"; -const std::string kMaxSpillBytes = "spark.gluten.sql.columnar.backend.velox.MaxSpillBytes"; -const std::string kSpillWriteBufferSize = "spark.gluten.sql.columnar.backend.velox.spillWriteBufferSize"; - -const std::string kSpillableReservationGrowthPct = - "spark.gluten.sql.columnar.backend.velox.spillableReservationGrowthPct"; -const std::string kSpillCompressionKind = "spark.io.compression.codec"; -const std::string kMaxPartialAggregationMemoryRatio = - "spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio"; -const std::string kMaxExtendedPartialAggregationMemoryRatio = - "spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio"; -const std::string kAbandonPartialAggregationMinPct = - "spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct"; -const std::string kAbandonPartialAggregationMinRows = - "spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows"; - -// execution -const std::string kBloomFilterExpectedNumItems = "spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems"; -const std::string kBloomFilterNumBits = "spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits"; -const std::string kBloomFilterMaxNumBits = "spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits"; -const std::string kVeloxSplitPreloadPerDriver = "spark.gluten.sql.columnar.backend.velox.SplitPreloadPerDriver"; - -// write fies -const std::string kMaxPartitions = "spark.gluten.sql.columnar.backend.velox.maxPartitionsPerWritersSession"; // metrics const std::string kDynamicFiltersProduced = "dynamicFiltersProduced"; diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h new file mode 100644 index 000000000000..a3112f83ee35 --- /dev/null +++ b/cpp/velox/config/VeloxConfig.h @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "config/GlutenConfig.h" + +namespace gluten { +// memory +const std::string kSpillStrategy = "spark.gluten.sql.columnar.backend.velox.spillStrategy"; +const std::string kSpillStrategyDefaultValue = "auto"; +const std::string kSpillThreadNum = "spark.gluten.sql.columnar.backend.velox.spillThreadNum"; +const uint32_t kSpillThreadNumDefaultValue = 0; +const std::string kAggregationSpillEnabled = "spark.gluten.sql.columnar.backend.velox.aggregationSpillEnabled"; +const std::string kJoinSpillEnabled = "spark.gluten.sql.columnar.backend.velox.joinSpillEnabled"; +const std::string kOrderBySpillEnabled = "spark.gluten.sql.columnar.backend.velox.orderBySpillEnabled"; + +// spill config +// refer to +// https://github.com/facebookincubator/velox/blob/95f3e80e77d046c12fbc79dc529366be402e9c2b/velox/docs/configs.rst#spilling +const std::string kMaxSpillLevel = "spark.gluten.sql.columnar.backend.velox.maxSpillLevel"; +const std::string kMaxSpillFileSize = "spark.gluten.sql.columnar.backend.velox.maxSpillFileSize"; +const std::string kSpillStartPartitionBit = "spark.gluten.sql.columnar.backend.velox.spillStartPartitionBit"; +const std::string kSpillPartitionBits = "spark.gluten.sql.columnar.backend.velox.spillPartitionBits"; +const std::string kMaxSpillRunRows = "spark.gluten.sql.columnar.backend.velox.MaxSpillRunRows"; +const std::string kMaxSpillBytes = "spark.gluten.sql.columnar.backend.velox.MaxSpillBytes"; +const std::string kSpillWriteBufferSize = "spark.gluten.sql.columnar.backend.velox.spillWriteBufferSize"; +const uint64_t kMaxSpillFileSizeDefault = 1L * 1024 * 1024 * 1024; + +const std::string kSpillableReservationGrowthPct = + "spark.gluten.sql.columnar.backend.velox.spillableReservationGrowthPct"; +const std::string kSpillCompressionKind = "spark.io.compression.codec"; +const std::string kMaxPartialAggregationMemoryRatio = + "spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio"; +const std::string kMaxExtendedPartialAggregationMemoryRatio = + "spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio"; +const std::string kAbandonPartialAggregationMinPct = + "spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct"; +const std::string kAbandonPartialAggregationMinRows = + "spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows"; + +// execution +const std::string kBloomFilterExpectedNumItems = "spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems"; +const std::string kBloomFilterNumBits = "spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits"; +const std::string kBloomFilterMaxNumBits = "spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits"; +const std::string kVeloxSplitPreloadPerDriver = "spark.gluten.sql.columnar.backend.velox.SplitPreloadPerDriver"; + +const std::string kEnableUserExceptionStacktrace = + "spark.gluten.sql.columnar.backend.velox.enableUserExceptionStacktrace"; +const bool kEnableUserExceptionStacktraceDefault = true; + +const std::string kEnableSystemExceptionStacktrace = + "spark.gluten.sql.columnar.backend.velox.enableSystemExceptionStacktrace"; +const bool kEnableSystemExceptionStacktraceDefault = true; + +const std::string kMemoryUseHugePages = "spark.gluten.sql.columnar.backend.velox.memoryUseHugePages"; +const bool kMemoryUseHugePagesDefault = false; + +const std::string kHiveConnectorId = "test-hive"; +const std::string kVeloxCacheEnabled = "spark.gluten.sql.columnar.backend.velox.cacheEnabled"; + +// memory cache +const std::string kVeloxMemCacheSize = "spark.gluten.sql.columnar.backend.velox.memCacheSize"; +const uint64_t kVeloxMemCacheSizeDefault = 1073741824; // 1G + +// ssd cache +const std::string kVeloxSsdCacheSize = "spark.gluten.sql.columnar.backend.velox.ssdCacheSize"; +const uint64_t kVeloxSsdCacheSizeDefault = 1073741824; // 1G +const std::string kVeloxSsdCachePath = "spark.gluten.sql.columnar.backend.velox.ssdCachePath"; +const std::string kVeloxSsdCachePathDefault = "/tmp/"; +const std::string kVeloxSsdCacheShards = "spark.gluten.sql.columnar.backend.velox.ssdCacheShards"; +const uint32_t kVeloxSsdCacheShardsDefault = 1; +const std::string kVeloxSsdCacheIOThreads = "spark.gluten.sql.columnar.backend.velox.ssdCacheIOThreads"; +const uint32_t kVeloxSsdCacheIOThreadsDefault = 1; +const std::string kVeloxSsdODirectEnabled = "spark.gluten.sql.columnar.backend.velox.ssdODirect"; + +// async +const std::string kVeloxIOThreads = "spark.gluten.sql.columnar.backend.velox.IOThreads"; +const uint32_t kVeloxIOThreadsDefault = 0; +const std::string kVeloxAsyncTimeoutOnTaskStopping = + "spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping"; +const int32_t kVeloxAsyncTimeoutOnTaskStoppingDefault = 30000; // 30s + +// udf +const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.udfLibraryPaths"; + +// backtrace allocation +const std::string kBacktraceAllocation = "spark.gluten.backtrace.allocation"; + +// VeloxShuffleReader print flag. +const std::string kVeloxShuffleReaderPrintFlag = "spark.gluten.velox.shuffleReaderPrintFlag"; + +const std::string kVeloxFileHandleCacheEnabled = "spark.gluten.sql.columnar.backend.velox.fileHandleCacheEnabled"; +const bool kVeloxFileHandleCacheEnabledDefault = false; + +/* configs for file read in velox*/ +const std::string kDirectorySizeGuess = "spark.gluten.sql.columnar.backend.velox.directorySizeGuess"; +const std::string kFilePreloadThreshold = "spark.gluten.sql.columnar.backend.velox.filePreloadThreshold"; +const std::string kPrefetchRowGroups = "spark.gluten.sql.columnar.backend.velox.prefetchRowGroups"; +const std::string kLoadQuantum = "spark.gluten.sql.columnar.backend.velox.loadQuantum"; +const std::string kMaxCoalescedDistanceBytes = "spark.gluten.sql.columnar.backend.velox.maxCoalescedDistanceBytes"; +const std::string kMaxCoalescedBytes = "spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes"; +const std::string kCachePrefetchMinPct = "spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct"; + +// write fies +const std::string kMaxPartitions = "spark.gluten.sql.columnar.backend.velox.maxPartitionsPerWritersSession"; + +const std::string kGlogVerboseLevel = "spark.gluten.sql.columnar.backend.velox.glogVerboseLevel"; +const uint32_t kGlogVerboseLevelDefault = 0; +const uint32_t kGlogVerboseLevelMaximum = 99; +const std::string kGlogSeverityLevel = "spark.gluten.sql.columnar.backend.velox.glogSeverityLevel"; +const uint32_t kGlogSeverityLevelDefault = 1; +} // namespace gluten From 888e1e24403a7d42a936586bc4563e143769ae17 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 16 May 2024 13:34:13 +0800 Subject: [PATCH 080/402] [VL] Daily Update Velox Version (2024_05_15) (#5748) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index d1e6054d810c..497befbe6018 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_14 +VELOX_BRANCH=2024_05_15 VELOX_HOME="" #Set on run gluten on HDFS From cb02cdb0a4095a8f194e62268147182afd48821c Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Thu, 16 May 2024 13:51:29 +0800 Subject: [PATCH 081/402] [VL] Enable length function for binary type (#5761) --- .../gluten/execution/ScalarFunctionsValidateSuite.scala | 8 ++++++++ cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 834e172f86bd..e88e9699a9d0 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -977,4 +977,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } } + + test("length") { + runQueryAndCompare( + "select length(c_comment), length(cast(c_comment as binary))" + + " from customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } } diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index fc8b912e0c62..51f39a3abdbe 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -191,12 +191,6 @@ bool SubstraitToVeloxPlanValidator::validateScalarFunction( return validateRound(scalarFunction, inputType); } else if (name == "extract") { return validateExtractExpr(params); - } else if (name == "char_length") { - VELOX_CHECK(types.size() == 1); - if (types[0] == "vbin") { - LOG_VALIDATION_MSG("Binary type is not supported in " + name); - return false; - } } else if (name == "map_from_arrays") { LOG_VALIDATION_MSG("map_from_arrays is not supported."); return false; From f925180018bb4fc770fac1e394aa0ec7cb1c2ad8 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Thu, 16 May 2024 14:43:22 +0800 Subject: [PATCH 082/402] [CORE] Remove wrong comment for JoinSelectionOverrides (#5730) --- .../scala/org/apache/gluten/extension/StrategyOverrides.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala index 28ce72546dcc..d016eacccf11 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala @@ -58,7 +58,6 @@ case class JoinSelectionOverrides(session: SparkSession) hint: JoinHint, forceShuffledHashJoin: Boolean): Seq[SparkPlan] = { if (isBroadcastStage(left) || isBroadcastStage(right)) { - // equal condition val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight Seq( BroadcastHashJoinExec( @@ -70,7 +69,6 @@ case class JoinSelectionOverrides(session: SparkSession) planLater(left), planLater(right))) } else { - // non equal condition // Generate BHJ here, avoid to do match in `JoinSelection` again. val isHintEmpty = hint.leftHint.isEmpty && hint.rightHint.isEmpty val buildSide = getBroadcastBuildSide(left, right, joinType, hint, !isHintEmpty, conf) From 0cbb7f2297a940047dfb788caac17cb2ad540356 Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 16 May 2024 15:09:44 +0800 Subject: [PATCH 083/402] [GLUTEN-5696] Add preprojection support for ArrowEvalPythonExec (#5697) --- .../velox/VeloxSparkPlanExecApi.scala | 8 +- .../python/ColumnarArrowEvalPythonExec.scala | 97 +++++++++++++++++-- .../python/ArrowEvalPythonExecSuite.scala | 43 +++++++- .../gluten/backendsapi/SparkPlanExecApi.scala | 4 + .../columnar/rewrite/PullOutPreProject.scala | 5 + .../RewriteSparkPlanRulesManager.scala | 2 + 6 files changed, 149 insertions(+), 10 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 4d41ed0c0a79..33ce1ee72550 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -31,7 +31,7 @@ import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode import org.apache.gluten.vectorized.{ColumnarBatchSerializer, ColumnarBatchSerializeResult} import org.apache.spark.{ShuffleDependency, SparkException} -import org.apache.spark.api.python.ColumnarArrowEvalPythonExec +import org.apache.spark.api.python.{ColumnarArrowEvalPythonExec, PullOutArrowEvalPythonPreProjectHelper} import org.apache.spark.rdd.RDD import org.apache.spark.serializer.Serializer import org.apache.spark.shuffle.{GenShuffleWriterParameters, GlutenShuffleWriterWrapper} @@ -53,6 +53,7 @@ import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{BuildSideRelation, HashedRelationBroadcastMode} import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.python.ArrowEvalPythonExec import org.apache.spark.sql.execution.utils.ExecUtil import org.apache.spark.sql.expression.{UDFExpression, UDFResolver, UserDefinedAggregateFunction} import org.apache.spark.sql.internal.SQLConf @@ -846,6 +847,11 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { PullOutGenerateProjectHelper.pullOutPostProject(generate) } + override def genPreProjectForArrowEvalPythonExec( + arrowEvalPythonExec: ArrowEvalPythonExec): SparkPlan = { + PullOutArrowEvalPythonPreProjectHelper.pullOutPreProject(arrowEvalPythonExec) + } + override def maybeCollapseTakeOrderedAndProject(plan: SparkPlan): SparkPlan = { // This to-top-n optimization assumes exchange operators were already placed in input plan. plan.transformUp { diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala index 77ef1c6422b2..d3112c97410d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala @@ -17,17 +17,18 @@ package org.apache.spark.api.python import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.exception.GlutenException import org.apache.gluten.extension.GlutenPlan import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.utils.Iterators +import org.apache.gluten.utils.{Iterators, PullOutProjectHelper} import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.{ContextAwareIterator, SparkEnv, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.python.{BasePythonRunnerShim, EvalPythonExec, PythonUDFRunner} +import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} +import org.apache.spark.sql.execution.python.{ArrowEvalPythonExec, BasePythonRunnerShim, EvalPythonExec, PythonUDFRunner} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructField, StructType} import org.apache.spark.sql.utils.{SparkArrowUtil, SparkSchemaUtil, SparkVectorUtil} @@ -41,6 +42,7 @@ import java.io.{DataInputStream, DataOutputStream} import java.net.Socket import java.util.concurrent.atomic.AtomicBoolean +import scala.collection.{mutable, Seq} import scala.collection.mutable.ArrayBuffer class ColumnarArrowPythonRunner( @@ -207,7 +209,6 @@ case class ColumnarArrowEvalPythonExec( extends EvalPythonExec with GlutenPlan { override def supportsColumnar: Boolean = true - // TODO: add additional projection support by pre-project // FIXME: incorrect metrics updater override protected def evaluate( @@ -221,6 +222,7 @@ case class ColumnarArrowEvalPythonExec( } private val sessionLocalTimeZone = conf.sessionLocalTimeZone + private def getPythonRunnerConfMap(conf: SQLConf): Map[String, String] = { val timeZoneConf = Seq(SQLConf.SESSION_LOCAL_TIMEZONE.key -> conf.sessionLocalTimeZone) val pandasColsByName = Seq( @@ -231,6 +233,7 @@ case class ColumnarArrowEvalPythonExec( conf.arrowSafeTypeConversion.toString) Map(timeZoneConf ++ pandasColsByName ++ arrowSafeTypeCheck: _*) } + private val pythonRunnerConf = getPythonRunnerConfMap(conf) protected def evaluateColumnar( @@ -279,16 +282,29 @@ case class ColumnarArrowEvalPythonExec( iter => val context = TaskContext.get() val (pyFuncs, inputs) = udfs.map(collectFunctions).unzip - // flatten all the arguments + // We only write the referred cols by UDFs to python worker. So we need + // get corresponding offsets val allInputs = new ArrayBuffer[Expression] val dataTypes = new ArrayBuffer[DataType] + val originalOffsets = new ArrayBuffer[Int] val argOffsets = inputs.map { input => input.map { e => - if (allInputs.exists(_.semanticEquals(e))) { + if (!e.isInstanceOf[AttributeReference]) { + throw new GlutenException( + "ColumnarArrowEvalPythonExec should only has [AttributeReference] inputs.") + } else if (allInputs.exists(_.semanticEquals(e))) { allInputs.indexWhere(_.semanticEquals(e)) } else { + var offset: Int = -1 + offset = child.output.indexWhere( + _.exprId.equals(e.asInstanceOf[AttributeReference].exprId)) + if (offset == -1) { + throw new GlutenException( + "ColumnarArrowEvalPythonExec can't find referred input col.") + } + originalOffsets += offset allInputs += e dataTypes += e.dataType allInputs.length - 1 @@ -299,15 +315,21 @@ case class ColumnarArrowEvalPythonExec( case (dt, i) => StructField(s"_$i", dt) }.toSeq) + val contextAwareIterator = new ContextAwareIterator(context, iter) val inputCbCache = new ArrayBuffer[ColumnarBatch]() val inputBatchIter = contextAwareIterator.map { inputCb => ColumnarBatches.ensureLoaded(ArrowBufferAllocators.contextInstance, inputCb) - // 0. cache input for later merge ColumnarBatches.retain(inputCb) + // 0. cache input for later merge inputCbCache += inputCb - inputCb + // We only need to pass the referred cols data to python worker for evaluation. + var colsForEval = new ArrayBuffer[ColumnVector]() + for (i <- originalOffsets) { + colsForEval += inputCb.column(i) + } + new ColumnarBatch(colsForEval.toArray, inputCb.numRows()) } val outputColumnarBatchIterator = @@ -335,6 +357,65 @@ case class ColumnarArrowEvalPythonExec( .create() } } + override protected def withNewChildInternal(newChild: SparkPlan): ColumnarArrowEvalPythonExec = copy(udfs, resultAttrs, newChild) } + +object PullOutArrowEvalPythonPreProjectHelper extends PullOutProjectHelper { + private def collectFunctions(udf: PythonUDF): (ChainedPythonFunctions, Seq[Expression]) = { + udf.children match { + case Seq(u: PythonUDF) => + val (chained, children) = collectFunctions(u) + (ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), children) + case children => + (ChainedPythonFunctions(Seq(udf.func)), udf.children) + } + } + + private def rewriteUDF( + udf: PythonUDF, + expressionMap: mutable.HashMap[Expression, NamedExpression]): PythonUDF = { + udf.children match { + case Seq(u: PythonUDF) => + udf + .withNewChildren(udf.children.toIndexedSeq.map { + func => rewriteUDF(func.asInstanceOf[PythonUDF], expressionMap) + }) + .asInstanceOf[PythonUDF] + case children => + val newUDFChildren = udf.children.map { + case literal: Literal => literal + case other => replaceExpressionWithAttribute(other, expressionMap) + } + udf.withNewChildren(newUDFChildren).asInstanceOf[PythonUDF] + } + } + + def pullOutPreProject(arrowEvalPythonExec: ArrowEvalPythonExec): SparkPlan = { + // pull out preproject + val (_, inputs) = arrowEvalPythonExec.udfs.map(collectFunctions).unzip + val expressionMap = new mutable.HashMap[Expression, NamedExpression]() + // flatten all the arguments + val allInputs = new ArrayBuffer[Expression] + for (input <- inputs) { + input.map { + e => + if (!allInputs.exists(_.semanticEquals(e))) { + allInputs += e + replaceExpressionWithAttribute(e, expressionMap) + } + } + } + if (!expressionMap.isEmpty) { + // Need preproject. + val preProject = ProjectExec( + eliminateProjectList(arrowEvalPythonExec.child.outputSet, expressionMap.values.toSeq), + arrowEvalPythonExec.child) + val newUDFs = arrowEvalPythonExec.udfs.map(f => rewriteUDF(f, expressionMap)) + arrowEvalPythonExec.copy(udfs = newUDFs, child = preProject) + } else { + arrowEvalPythonExec + } + } +} diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/python/ArrowEvalPythonExecSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/python/ArrowEvalPythonExecSuite.scala index 2193448b4d22..1c3e33262c7a 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/python/ArrowEvalPythonExecSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/python/ArrowEvalPythonExecSuite.scala @@ -39,7 +39,7 @@ class ArrowEvalPythonExecSuite extends WholeStageTransformerSuite { .set("spark.executor.cores", "1") } - test("arrow_udf test") { + test("arrow_udf test: without projection") { lazy val base = Seq(("1", 1), ("1", 2), ("2", 1), ("2", 2), ("3", 1), ("3", 2), ("0", 1), ("3", 0)) .toDF("a", "b") @@ -58,4 +58,45 @@ class ArrowEvalPythonExecSuite extends WholeStageTransformerSuite { checkSparkOperatorMatch[ColumnarArrowEvalPythonExec](df2) checkAnswer(df2, expected) } + + test("arrow_udf test: with unrelated projection") { + lazy val base = + Seq(("1", 1), ("1", 2), ("2", 1), ("2", 2), ("3", 1), ("3", 2), ("0", 1), ("3", 0)) + .toDF("a", "b") + lazy val expected = Seq( + ("1", 1, "1", 2), + ("1", 2, "1", 4), + ("2", 1, "2", 2), + ("2", 2, "2", 4), + ("3", 1, "3", 2), + ("3", 2, "3", 4), + ("0", 1, "0", 2), + ("3", 0, "3", 0) + ).toDF("a", "b", "p_a", "d_b") + + val df = base.withColumn("p_a", pyarrowTestUDF(base("a"))).withColumn("d_b", base("b") * 2) + checkSparkOperatorMatch[ColumnarArrowEvalPythonExec](df) + checkAnswer(df, expected) + } + + test("arrow_udf test: with preprojection") { + lazy val base = + Seq(("1", 1), ("1", 2), ("2", 1), ("2", 2), ("3", 1), ("3", 2), ("0", 1), ("3", 0)) + .toDF("a", "b") + lazy val expected = Seq( + ("1", 1, 2, "1", 2), + ("1", 2, 4, "1", 4), + ("2", 1, 2, "2", 2), + ("2", 2, 4, "2", 4), + ("3", 1, 2, "3", 2), + ("3", 2, 4, "3", 4), + ("0", 1, 2, "0", 2), + ("3", 0, 0, "3", 0) + ).toDF("a", "b", "d_b", "p_a", "p_b") + val df = base + .withColumn("d_b", base("b") * 2) + .withColumn("p_a", pyarrowTestUDF(base("a"))) + .withColumn("p_b", pyarrowTestUDF(base("b") * 2)) + checkAnswer(df, expected) + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index c2c733070688..8df74bb88bfe 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -43,6 +43,7 @@ import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, FileScan} import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.joins.BuildSideRelation import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.python.ArrowEvalPythonExec import org.apache.spark.sql.hive.HiveTableScanExecTransformer import org.apache.spark.sql.types.{LongType, NullType, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch @@ -745,6 +746,9 @@ trait SparkPlanExecApi { def genPostProjectForGenerate(generate: GenerateExec): SparkPlan + def genPreProjectForArrowEvalPythonExec(arrowEvalPythonExec: ArrowEvalPythonExec): SparkPlan = + arrowEvalPythonExec + def maybeCollapseTakeOrderedAndProject(plan: SparkPlan): SparkPlan = plan def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = false diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala index 64d4f273622c..50dc55423605 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, Partial} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, TypedAggregateExpression} +import org.apache.spark.sql.execution.python.ArrowEvalPythonExec import org.apache.spark.sql.execution.window.{WindowExec, WindowGroupLimitExecShim} import scala.collection.mutable @@ -226,6 +227,10 @@ object PullOutPreProject extends RewriteSingleNode with PullOutProjectHelper { case generate: GenerateExec => BackendsApiManager.getSparkPlanExecApiInstance.genPreProjectForGenerate(generate) + case arrowEvalPythonExec: ArrowEvalPythonExec => + BackendsApiManager.getSparkPlanExecApiInstance.genPreProjectForArrowEvalPythonExec( + arrowEvalPythonExec) + case _ => plan } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala index 5fd728eca65a..ac663314bead 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.execution.joins.BaseJoinExec +import org.apache.spark.sql.execution.python.ArrowEvalPythonExec import org.apache.spark.sql.execution.window.WindowExec case class RewrittenNodeWall(originalChild: SparkPlan) extends LeafExecNode { @@ -60,6 +61,7 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] case _: ExpandExec => true case _: GenerateExec => true case plan if SparkShimLoader.getSparkShims.isWindowGroupLimitExec(plan) => true + case _: ArrowEvalPythonExec => true case _ => false } } From 6a110e5e60d5f195293119f42a58c9f6911c987c Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Thu, 16 May 2024 16:02:54 +0800 Subject: [PATCH 084/402] [GLUTEN-5414] [VL] Support datasource v2 scan csv (#5717) --- .../velox/VeloxSparkPlanExecApi.scala | 2 + .../datasource/ArrowCSVFileFormat.scala | 277 ++++++++++-------- .../datasource/ArrowConvertorRule.scala | 49 +++- .../v2/ArrowCSVPartitionReaderFactory.scala | 144 +++++++++ .../gluten/datasource/v2/ArrowCSVScan.scala | 76 +++++ .../datasource/v2/ArrowCSVScanBuilder.scala | 44 +++ .../gluten/datasource/v2/ArrowCSVTable.scala | 68 +++++ .../datasource/v2/ArrowBatchScanExec.scala | 48 +++ .../extension/ArrowScanReplaceRule.scala | 7 +- .../gluten/execution/TestOperator.scala | 22 +- .../gluten/columnarbatch/ColumnarBatches.java | 4 +- .../org/apache/gluten/utils/ArrowUtil.scala | 35 ++- .../utils/velox/VeloxTestSettings.scala | 9 +- .../utils/velox/VeloxTestSettings.scala | 9 + .../utils/velox/VeloxTestSettings.scala | 9 + .../utils/velox/VeloxTestSettings.scala | 9 + .../datasources/csv/GlutenCSVSuite.scala | 1 + .../datasources/v2/BatchScanExecShim.scala | 4 + .../datasources/v2/BatchScanExecShim.scala | 6 + .../datasources/v2/BatchScanExecShim.scala | 8 + .../datasources/v2/BatchScanExecShim.scala | 8 + 21 files changed, 680 insertions(+), 159 deletions(-) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScan.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScanBuilder.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 33ce1ee72550..f54bf9b3f61e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -21,6 +21,7 @@ import org.apache.gluten.backendsapi.SparkPlanExecApi import org.apache.gluten.datasource.ArrowConvertorRule import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ +import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} @@ -869,6 +870,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { override def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = plan match { case _: ArrowFileSourceScanExec => true + case _: ArrowBatchScanExec => true case _ => false } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala index c05af24ff611..0f6813d8fc6a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala @@ -20,6 +20,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.SchemaMismatchException import org.apache.gluten.execution.RowToVeloxColumnarExec import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool import org.apache.gluten.utils.{ArrowUtil, Iterators} import org.apache.gluten.vectorized.ArrowWritableColumnVector @@ -41,6 +42,7 @@ import org.apache.spark.util.SerializableConfiguration import org.apache.arrow.dataset.file.FileSystemDatasetFactory import org.apache.arrow.dataset.scanner.ScanOptions +import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.VectorUnloader import org.apache.arrow.vector.types.pojo.Schema import org.apache.hadoop.conf.Configuration @@ -66,55 +68,127 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging sparkSession: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Option[StructType] = { - ArrowUtil.readSchema(files, fileFormat) + ArrowUtil.readSchema( + files, + fileFormat, + ArrowBufferAllocators.contextInstance(), + ArrowNativeMemoryPool.arrowPool("infer schema")) } override def supportBatch(sparkSession: SparkSession, dataSchema: StructType): Boolean = true - private def checkHeader( - file: PartitionedFile, + override def buildReaderWithPartitionValues( + sparkSession: SparkSession, dataSchema: StructType, + partitionSchema: StructType, requiredSchema: StructType, - parsedOptions: CSVOptions, - actualFilters: Seq[Filter], - conf: Configuration): Unit = { - val isStartOfFile = file.start == 0 - if (!isStartOfFile) { - return - } - val actualDataSchema = StructType( - dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) - val actualRequiredSchema = StructType( - requiredSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) - val parser = - new UnivocityParser(actualDataSchema, actualRequiredSchema, parsedOptions, actualFilters) - val schema = if (parsedOptions.columnPruning) actualRequiredSchema else actualDataSchema - val headerChecker = new CSVHeaderChecker( - schema, - parsedOptions, - source = s"CSV file: ${file.filePath}", - isStartOfFile) - - val lines = { - val linesReader = - new HadoopFileLinesReader(file, parser.options.lineSeparatorInRead, conf) - Option(TaskContext.get()) - .foreach(_.addTaskCompletionListener[Unit](_ => linesReader.close())) - linesReader.map { - line => new String(line.getBytes, 0, line.getLength, parser.options.charset) + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { + val sqlConf = sparkSession.sessionState.conf + val broadcastedHadoopConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) + val batchSize = sqlConf.columnBatchSize + val caseSensitive = sqlConf.caseSensitiveAnalysis + val columnPruning = sqlConf.csvColumnPruning && + !requiredSchema.exists(_.name == sparkSession.sessionState.conf.columnNameOfCorruptRecord) + val parsedOptions = new CSVOptions( + options, + columnPruning, + sparkSession.sessionState.conf.sessionLocalTimeZone, + sparkSession.sessionState.conf.columnNameOfCorruptRecord) + val actualFilters = + filters.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord)) + (file: PartitionedFile) => { + ArrowCSVFileFormat.checkHeader( + file, + dataSchema, + requiredSchema, + parsedOptions, + actualFilters, + broadcastedHadoopConf.value.value) + val factory = + ArrowUtil.makeArrowDiscovery( + URLDecoder.decode(file.filePath.toString, "UTF-8"), + fileFormat, + ArrowBufferAllocators.contextInstance(), + ArrowNativeMemoryPool.arrowPool("FileSystemDatasetFactory") + ) + // todo predicate validation / pushdown + val fileFields = factory.inspect().getFields.asScala + // TODO: support array/map/struct types in out-of-order schema reading. + try { + val actualReadFields = + ArrowUtil.getRequestedField(requiredSchema, fileFields, caseSensitive) + ArrowCSVFileFormat + .readArrow( + ArrowBufferAllocators.contextInstance(), + file, + actualReadFields, + caseSensitive, + requiredSchema, + partitionSchema, + factory, + batchSize) + .asInstanceOf[Iterator[InternalRow]] + } catch { + case e: SchemaMismatchException => + logWarning(e.getMessage) + val iter = ArrowCSVFileFormat.fallbackReadVanilla( + dataSchema, + requiredSchema, + broadcastedHadoopConf.value.value, + parsedOptions, + file, + actualFilters, + columnPruning) + val (schema, rows) = + ArrowCSVFileFormat.withPartitionValue(requiredSchema, partitionSchema, iter, file) + ArrowCSVFileFormat + .rowToColumn(schema, batchSize, rows) + .asInstanceOf[Iterator[InternalRow]] + case d: Exception => throw d } + } - CSVHeaderCheckerHelper.checkHeaderColumnNames(headerChecker, lines, parser.tokenizer) } - private def readArrow( + override def vectorTypes( + requiredSchema: StructType, + partitionSchema: StructType, + sqlConf: SQLConf): Option[Seq[String]] = { + Option( + Seq.fill(requiredSchema.fields.length + partitionSchema.fields.length)( + classOf[ArrowWritableColumnVector].getName + )) + } + + override def shortName(): String = "arrowcsv" + + override def hashCode(): Int = getClass.hashCode() + + override def equals(other: Any): Boolean = other.isInstanceOf[ArrowCSVFileFormat] + + override def prepareWrite( + sparkSession: SparkSession, + job: _root_.org.apache.hadoop.mapreduce.Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + throw new UnsupportedOperationException() + } +} + +object ArrowCSVFileFormat { + + def readArrow( + allocator: BufferAllocator, file: PartitionedFile, actualReadFields: Schema, caseSensitive: Boolean, requiredSchema: StructType, partitionSchema: StructType, factory: FileSystemDatasetFactory, - batchSize: Int): Iterator[InternalRow] = { + batchSize: Int): Iterator[ColumnarBatch] = { val compare = ArrowUtil.compareStringFunc(caseSensitive) val actualReadFieldNames = actualReadFields.getFields.asScala.map(_.getName).toArray val actualReadSchema = new StructType( @@ -147,7 +221,9 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging override def next: ColumnarBatch = { val root = reader.getVectorSchemaRoot val unloader = new VectorUnloader(root) + val batch = ArrowUtil.loadBatch( + allocator, unloader.getRecordBatch, actualReadSchema, requiredSchema, @@ -166,13 +242,48 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging } .recyclePayload(_.close()) .create() - .asInstanceOf[Iterator[InternalRow]] } - private def rowToColumn( + def checkHeader( + file: PartitionedFile, + dataSchema: StructType, + requiredSchema: StructType, + parsedOptions: CSVOptions, + actualFilters: Seq[Filter], + conf: Configuration): Unit = { + val isStartOfFile = file.start == 0 + if (!isStartOfFile) { + return + } + val actualDataSchema = StructType( + dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val actualRequiredSchema = StructType( + requiredSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val parser = + new UnivocityParser(actualDataSchema, actualRequiredSchema, parsedOptions, actualFilters) + val schema = if (parsedOptions.columnPruning) actualRequiredSchema else actualDataSchema + val headerChecker = new CSVHeaderChecker( + schema, + parsedOptions, + source = s"CSV file: ${file.filePath}", + isStartOfFile) + + val lines = { + val linesReader = + new HadoopFileLinesReader(file, parser.options.lineSeparatorInRead, conf) + Option(TaskContext.get()) + .foreach(_.addTaskCompletionListener[Unit](_ => linesReader.close())) + linesReader.map { + line => new String(line.getBytes, 0, line.getLength, parser.options.charset) + } + } + CSVHeaderCheckerHelper.checkHeaderColumnNames(headerChecker, lines, parser.tokenizer) + } + + def rowToColumn( schema: StructType, batchSize: Int, - it: Iterator[InternalRow]): Iterator[InternalRow] = { + it: Iterator[InternalRow]): Iterator[ColumnarBatch] = { // note, these metrics are unused but just make `RowToVeloxColumnarExec` happy val numInputRows = new SQLMetric("numInputRows") val numOutputBatches = new SQLMetric("numOutputBatches") @@ -187,7 +298,6 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging ) veloxBatch .map(v => ColumnarBatches.ensureLoaded(ArrowBufferAllocators.contextInstance(), v)) - .asInstanceOf[Iterator[InternalRow]] } private def toAttribute(field: StructField): AttributeReference = @@ -197,7 +307,7 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging schema.map(toAttribute) } - private def withPartitionValue( + def withPartitionValue( requiredSchema: StructType, partitionSchema: StructType, iter: Iterator[InternalRow], @@ -223,7 +333,7 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging } } - private def fallbackReadVanilla( + def fallbackReadVanilla( dataSchema: StructType, requiredSchema: StructType, conf: Configuration, @@ -246,93 +356,4 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging isStartOfFile) CSVDataSource(parsedOptions).readFile(conf, file, parser, headerChecker, requiredSchema) } - - override def buildReaderWithPartitionValues( - sparkSession: SparkSession, - dataSchema: StructType, - partitionSchema: StructType, - requiredSchema: StructType, - filters: Seq[Filter], - options: Map[String, String], - hadoopConf: Configuration): PartitionedFile => Iterator[InternalRow] = { - val sqlConf = sparkSession.sessionState.conf - val broadcastedHadoopConf = - sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) - val batchSize = sqlConf.columnBatchSize - val caseSensitive = sqlConf.caseSensitiveAnalysis - val columnPruning = sqlConf.csvColumnPruning && - !requiredSchema.exists(_.name == sparkSession.sessionState.conf.columnNameOfCorruptRecord) - val parsedOptions = new CSVOptions( - options, - columnPruning, - sparkSession.sessionState.conf.sessionLocalTimeZone, - sparkSession.sessionState.conf.columnNameOfCorruptRecord) - val actualFilters = - filters.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord)) - (file: PartitionedFile) => { - checkHeader( - file, - dataSchema, - requiredSchema, - parsedOptions, - actualFilters, - broadcastedHadoopConf.value.value) - val factory = - ArrowUtil.makeArrowDiscovery(URLDecoder.decode(file.filePath.toString, "UTF-8"), fileFormat) - // todo predicate validation / pushdown - val fileFields = factory.inspect().getFields.asScala - // TODO: support array/map/struct types in out-of-order schema reading. - try { - val actualReadFields = - ArrowUtil.getRequestedField(requiredSchema, fileFields, caseSensitive) - readArrow( - file, - actualReadFields, - caseSensitive, - requiredSchema, - partitionSchema, - factory, - batchSize) - } catch { - case e: SchemaMismatchException => - logWarning(e.getMessage) - val iter = fallbackReadVanilla( - dataSchema, - requiredSchema, - broadcastedHadoopConf.value.value, - parsedOptions, - file, - actualFilters, - columnPruning) - val (schema, rows) = withPartitionValue(requiredSchema, partitionSchema, iter, file) - rowToColumn(schema, batchSize, rows) - case d: Exception => throw d - } - - } - } - - override def vectorTypes( - requiredSchema: StructType, - partitionSchema: StructType, - sqlConf: SQLConf): Option[Seq[String]] = { - Option( - Seq.fill(requiredSchema.fields.length + partitionSchema.fields.length)( - classOf[ArrowWritableColumnVector].getName - )) - } - - override def shortName(): String = "arrowcsv" - - override def hashCode(): Int = getClass.hashCode() - - override def equals(other: Any): Boolean = other.isInstanceOf[ArrowCSVFileFormat] - - override def prepareWrite( - sparkSession: SparkSession, - job: _root_.org.apache.hadoop.mapreduce.Job, - options: Map[String, String], - dataSchema: StructType): OutputWriterFactory = { - throw new UnsupportedOperationException() - } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala index e29313a3809e..dab1ffd3b9e3 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala @@ -17,6 +17,7 @@ package org.apache.gluten.datasource import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.datasource.v2.ArrowCSVTable import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.annotation.Experimental @@ -27,11 +28,15 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.PermissiveMode import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkSchemaUtil import java.nio.charset.StandardCharsets +import scala.collection.convert.ImplicitConversions.`map AsScala` + @Experimental case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { @@ -39,27 +44,49 @@ case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { return plan } plan.resolveOperators { - // Read path case l @ LogicalRelation( r @ HadoopFsRelation(_, _, dataSchema, _, _: CSVFileFormat, options), _, _, - _) => - val csvOptions = new CSVOptions( + _) if validate(session, dataSchema, options) => + l.copy(relation = r.copy(fileFormat = new ArrowCSVFileFormat())(session)) + case d @ DataSourceV2Relation( + t @ CSVTable( + name, + sparkSession, + options, + paths, + userSpecifiedSchema, + fallbackFileFormat), + _, + _, + _, + _) if validate(session, t.dataSchema, options.asCaseSensitiveMap().toMap) => + d.copy(table = ArrowCSVTable( + "arrow" + name, + sparkSession, options, - columnPruning = session.sessionState.conf.csvColumnPruning, - session.sessionState.conf.sessionLocalTimeZone) - if ( - checkSchema(dataSchema) && - checkCsvOptions(csvOptions, session.sessionState.conf.sessionLocalTimeZone) - ) { - l.copy(relation = r.copy(fileFormat = new ArrowCSVFileFormat())(session)) - } else l + paths, + userSpecifiedSchema, + fallbackFileFormat)) case r => r } } + private def validate( + session: SparkSession, + dataSchema: StructType, + options: Map[String, String]): Boolean = { + val csvOptions = new CSVOptions( + options, + columnPruning = session.sessionState.conf.csvColumnPruning, + session.sessionState.conf.sessionLocalTimeZone) + checkSchema(dataSchema) && + checkCsvOptions(csvOptions, session.sessionState.conf.sessionLocalTimeZone) && + dataSchema.nonEmpty + } + private def checkCsvOptions(csvOptions: CSVOptions, timeZone: String): Boolean = { csvOptions.headerFlag && !csvOptions.multiLine && csvOptions.delimiter == "," && csvOptions.quote == '\"' && diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala new file mode 100644 index 000000000000..ddc7f797fb93 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource.v2 + +import org.apache.gluten.datasource.ArrowCSVFileFormat +import org.apache.gluten.exception.SchemaMismatchException +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool +import org.apache.gluten.utils.ArrowUtil + +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.csv.CSVOptions +import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader} +import org.apache.spark.sql.execution.datasources.PartitionedFile +import org.apache.spark.sql.execution.datasources.v2.FilePartitionReaderFactory +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.{SerializableConfiguration, TaskResources} + +import org.apache.arrow.dataset.file.FileFormat + +import java.net.URLDecoder + +import scala.collection.JavaConverters.asScalaBufferConverter + +case class ArrowCSVPartitionReaderFactory( + sqlConf: SQLConf, + broadcastedConf: Broadcast[SerializableConfiguration], + dataSchema: StructType, + readDataSchema: StructType, + readPartitionSchema: StructType, + options: CSVOptions, + filters: Seq[Filter]) + extends FilePartitionReaderFactory + with Logging { + + private val batchSize = sqlConf.parquetVectorizedReaderBatchSize + private val caseSensitive: Boolean = sqlConf.caseSensitiveAnalysis + private val csvColumnPruning: Boolean = sqlConf.csvColumnPruning + + override def supportColumnarReads(partition: InputPartition): Boolean = true + + override def buildReader(partitionedFile: PartitionedFile): PartitionReader[InternalRow] = { + // disable row based read + throw new UnsupportedOperationException + } + + override def buildColumnarReader( + partitionedFile: PartitionedFile): PartitionReader[ColumnarBatch] = { + val actualDataSchema = StructType( + dataSchema.filterNot(_.name == options.columnNameOfCorruptRecord)) + val actualReadDataSchema = StructType( + readDataSchema.filterNot(_.name == options.columnNameOfCorruptRecord)) + ArrowCSVFileFormat.checkHeader( + partitionedFile, + actualDataSchema, + actualReadDataSchema, + options, + filters, + broadcastedConf.value.value) + val (allocator, pool) = if (!TaskResources.inSparkTask()) { + TaskResources.runUnsafe( + ( + ArrowBufferAllocators.contextInstance(), + ArrowNativeMemoryPool.arrowPool("FileSystemFactory")) + ) + } else { + ( + ArrowBufferAllocators.contextInstance(), + ArrowNativeMemoryPool.arrowPool("FileSystemFactory")) + } + val factory = ArrowUtil.makeArrowDiscovery( + URLDecoder.decode(partitionedFile.filePath.toString(), "UTF-8"), + FileFormat.CSV, + allocator, + pool) + val parquetFileFields = factory.inspect().getFields.asScala + // TODO: support array/map/struct types in out-of-order schema reading. + val iter = + try { + val actualReadFields = + ArrowUtil.getRequestedField(readDataSchema, parquetFileFields, caseSensitive) + ArrowCSVFileFormat.readArrow( + allocator, + partitionedFile, + actualReadFields, + caseSensitive, + readDataSchema, + readPartitionSchema, + factory, + batchSize) + } catch { + case e: SchemaMismatchException => + logWarning(e.getMessage) + val iter = ArrowCSVFileFormat.fallbackReadVanilla( + dataSchema, + readDataSchema, + broadcastedConf.value.value, + options, + partitionedFile, + filters, + csvColumnPruning) + val (schema, rows) = ArrowCSVFileFormat.withPartitionValue( + readDataSchema, + readPartitionSchema, + iter, + partitionedFile) + ArrowCSVFileFormat.rowToColumn(schema, batchSize, rows) + case d: Exception => throw d + } + + new PartitionReader[ColumnarBatch] { + + override def next(): Boolean = { + iter.hasNext + } + + override def get(): ColumnarBatch = { + iter.next() + } + + override def close(): Unit = {} + } + } + +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScan.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScan.scala new file mode 100644 index 000000000000..ce3f84770464 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScan.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource.v2 + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.csv.CSVOptions +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.connector.read.PartitionReaderFactory +import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex +import org.apache.spark.sql.execution.datasources.v2.FileScan +import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.util.SerializableConfiguration + +import org.apache.hadoop.fs.Path + +import scala.collection.JavaConverters.mapAsScalaMapConverter + +case class ArrowCSVScan( + sparkSession: SparkSession, + fileIndex: PartitioningAwareFileIndex, + dataSchema: StructType, + readDataSchema: StructType, + readPartitionSchema: StructType, + pushedFilters: Array[Filter], + options: CaseInsensitiveStringMap, + partitionFilters: Seq[Expression] = Seq.empty, + dataFilters: Seq[Expression] = Seq.empty) + extends FileScan { + + private lazy val parsedOptions: CSVOptions = new CSVOptions( + options.asScala.toMap, + columnPruning = sparkSession.sessionState.conf.csvColumnPruning, + sparkSession.sessionState.conf.sessionLocalTimeZone, + sparkSession.sessionState.conf.columnNameOfCorruptRecord + ) + + override def isSplitable(path: Path): Boolean = { + false + } + + override def createReaderFactory(): PartitionReaderFactory = { + val caseSensitiveMap = options.asCaseSensitiveMap().asScala.toMap + val hconf = sparkSession.sessionState.newHadoopConfWithOptions(caseSensitiveMap) + val broadcastedConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hconf)) + val actualFilters = + pushedFilters.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord)) + ArrowCSVPartitionReaderFactory( + sparkSession.sessionState.conf, + broadcastedConf, + dataSchema, + readDataSchema, + readPartitionSchema, + parsedOptions, + actualFilters) + } + + def withFilters(partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): FileScan = + this.copy(partitionFilters = partitionFilters, dataFilters = dataFilters) +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScanBuilder.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScanBuilder.scala new file mode 100644 index 000000000000..2b3991fe2984 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVScanBuilder.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource.v2 + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.read.Scan +import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex +import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +case class ArrowCSVScanBuilder( + sparkSession: SparkSession, + fileIndex: PartitioningAwareFileIndex, + schema: StructType, + dataSchema: StructType, + options: CaseInsensitiveStringMap) + extends FileScanBuilder(sparkSession, fileIndex, dataSchema) { + + override def build(): Scan = { + ArrowCSVScan( + sparkSession, + fileIndex, + dataSchema, + readDataSchema(), + readPartitionSchema(), + Array.empty, + options) + } +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala new file mode 100644 index 000000000000..aa7f737f9cfc --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource.v2 + +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool +import org.apache.gluten.utils.ArrowUtil + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.connector.read.ScanBuilder +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.FileTable +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.util.TaskResources + +import org.apache.hadoop.fs.FileStatus + +case class ArrowCSVTable( + name: String, + sparkSession: SparkSession, + options: CaseInsensitiveStringMap, + paths: Seq[String], + userSpecifiedSchema: Option[StructType], + fallbackFileFormat: Class[_ <: FileFormat]) + extends FileTable(sparkSession, options, paths, userSpecifiedSchema) { + + override def inferSchema(files: Seq[FileStatus]): Option[StructType] = { + val (allocator, pool) = if (!TaskResources.inSparkTask()) { + TaskResources.runUnsafe( + (ArrowBufferAllocators.contextInstance(), ArrowNativeMemoryPool.arrowPool("inferSchema")) + ) + } else { + (ArrowBufferAllocators.contextInstance(), ArrowNativeMemoryPool.arrowPool("inferSchema")) + } + ArrowUtil.readSchema( + files.head, + org.apache.arrow.dataset.file.FileFormat.CSV, + allocator, + pool + ) + } + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { + ArrowCSVScanBuilder(sparkSession, fileIndex, schema, dataSchema, options) + } + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + throw new UnsupportedOperationException + } + + override def formatName: String = "arrowcsv" +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala new file mode 100644 index 000000000000..3c1c538207c5 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution.datasource.v2 + +import org.apache.gluten.extension.GlutenPlan + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.physical.Partitioning +import org.apache.spark.sql.connector.read.{Batch, PartitionReaderFactory, Scan} +import org.apache.spark.sql.execution.datasources.v2.{ArrowBatchScanExecShim, BatchScanExec} + +case class ArrowBatchScanExec(original: BatchScanExec) + extends ArrowBatchScanExecShim(original) + with GlutenPlan { + + @transient lazy val batch: Batch = original.batch + + override lazy val readerFactory: PartitionReaderFactory = original.readerFactory + + override lazy val inputRDD: RDD[InternalRow] = original.inputRDD + + override def outputPartitioning: Partitioning = original.outputPartitioning + + override def scan: Scan = original.scan + + override def doCanonicalize(): ArrowBatchScanExec = + this.copy(original = original.doCanonicalize()) + + override def nodeName: String = "Arrow" + original.nodeName + + override def output: Seq[Attribute] = original.output +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala index 2b7c4b1da91b..adfc6ca742c9 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala @@ -17,18 +17,23 @@ package org.apache.gluten.extension import org.apache.gluten.datasource.ArrowCSVFileFormat +import org.apache.gluten.datasource.v2.ArrowCSVScan +import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{ArrowFileSourceScanExec, FileSourceScanExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec case class ArrowScanReplaceRule(spark: SparkSession) extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { plan.transformUp { case plan: FileSourceScanExec if plan.relation.fileFormat.isInstanceOf[ArrowCSVFileFormat] => ArrowFileSourceScanExec(plan) + case plan: BatchScanExec if plan.scan.isInstanceOf[ArrowCSVScan] => + ArrowBatchScanExec(plan) + case plan: BatchScanExec => plan case p => p } - } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index bccb06a130ae..0872ac798382 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -18,6 +18,7 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.datasource.ArrowCSVFileFormat +import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf @@ -491,7 +492,6 @@ class TestOperator extends VeloxWholeStageTransformerSuite { runQueryAndCompare("select * from student") { df => val plan = df.queryExecution.executedPlan - print(plan) assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) val scan = plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).toList.head @@ -538,6 +538,26 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + test("csv scan datasource v2") { + withSQLConf("spark.sql.sources.useV1SourceList" -> "") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + runQueryAndCompare("select * from student") { + checkGlutenOperatorMatch[ArrowBatchScanExec] + } + runQueryAndCompare("select * from student where Name = 'Peter'") { + df => + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isEmpty) + assert(plan.find(s => s.isInstanceOf[ArrowBatchScanExec]).isDefined) + } + } + } + test("test OneRowRelation") { val df = sql("SELECT 1") checkAnswer(df, Row(1)) diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java index 624428dcba19..e2cfa335d5c6 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java @@ -19,7 +19,6 @@ import org.apache.gluten.exception.GlutenException; import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.Runtimes; -import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; import org.apache.gluten.memory.nmm.NativeMemoryManager; import org.apache.gluten.utils.ArrowAbiUtil; import org.apache.gluten.utils.ArrowUtil; @@ -221,8 +220,7 @@ private static ColumnarBatch offload(BufferAllocator allocator, ColumnarBatch in final Runtime runtime = Runtimes.contextInstance(); try (ArrowArray cArray = ArrowArray.allocateNew(allocator); ArrowSchema cSchema = ArrowSchema.allocateNew(allocator)) { - ArrowAbiUtil.exportFromSparkColumnarBatch( - ArrowBufferAllocators.contextInstance(), input, cSchema, cArray); + ArrowAbiUtil.exportFromSparkColumnarBatch(allocator, input, cSchema, cArray); long handle = ColumnarBatchJniWrapper.forRuntime(runtime) .createWithArrowArray(cSchema.memoryAddress(), cArray.memoryAddress()); diff --git a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala index 26bebcfae713..99eb72c70ea3 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala @@ -17,8 +17,6 @@ package org.apache.gluten.utils import org.apache.gluten.exception.SchemaMismatchException -import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.internal.Logging @@ -34,6 +32,7 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.arrow.c.{ArrowSchema, CDataDictionaryProvider, Data} import org.apache.arrow.dataset.file.{FileFormat, FileSystemDatasetFactory} +import org.apache.arrow.dataset.jni.NativeMemoryPool import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.ipc.message.ArrowRecordBatch import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} @@ -140,19 +139,22 @@ object ArrowUtil extends Logging { rewritten.toString } - def makeArrowDiscovery(encodedUri: String, format: FileFormat): FileSystemDatasetFactory = { - val allocator = ArrowBufferAllocators.contextInstance() - val factory = new FileSystemDatasetFactory( - allocator, - ArrowNativeMemoryPool.arrowPool("FileSystemDatasetFactory"), - format, - rewriteUri(encodedUri)) + def makeArrowDiscovery( + encodedUri: String, + format: FileFormat, + allocator: BufferAllocator, + pool: NativeMemoryPool): FileSystemDatasetFactory = { + val factory = new FileSystemDatasetFactory(allocator, pool, format, rewriteUri(encodedUri)) factory } - def readSchema(file: FileStatus, format: FileFormat): Option[StructType] = { + def readSchema( + file: FileStatus, + format: FileFormat, + allocator: BufferAllocator, + pool: NativeMemoryPool): Option[StructType] = { val factory: FileSystemDatasetFactory = - makeArrowDiscovery(file.getPath.toString, format) + makeArrowDiscovery(file.getPath.toString, format, allocator, pool) val schema = factory.inspect() try { Option(SparkSchemaUtil.fromArrowSchema(schema)) @@ -161,12 +163,16 @@ object ArrowUtil extends Logging { } } - def readSchema(files: Seq[FileStatus], format: FileFormat): Option[StructType] = { + def readSchema( + files: Seq[FileStatus], + format: FileFormat, + allocator: BufferAllocator, + pool: NativeMemoryPool): Option[StructType] = { if (files.isEmpty) { throw new IllegalArgumentException("No input file specified") } - readSchema(files.head, format) + readSchema(files.head, format, allocator, pool) } def compareStringFunc(caseSensitive: Boolean): (String, String) => Boolean = { @@ -254,6 +260,7 @@ object ArrowUtil extends Logging { } def loadBatch( + allocator: BufferAllocator, input: ArrowRecordBatch, dataSchema: StructType, requiredSchema: StructType, @@ -267,7 +274,7 @@ object ArrowUtil extends Logging { rowCount, SparkSchemaUtil.toArrowSchema(dataSchema), input, - ArrowBufferAllocators.contextInstance()) + allocator) } finally { input.close() } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index dbd7dc187ba5..366796a57465 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -428,8 +428,15 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-23786: warning should be printed if CSV header doesn't conform to schema") // file cars.csv include null string, Arrow not support to read .exclude("DDL test with schema") - // file cars.csv include null string, Arrow not support to read .exclude("old csv data source name works") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch + // Early Filter and Projection Push-Down generated an invalid plan + .exclude("SPARK-26208: write and read empty data to csv file with headers") enableSuite[GlutenCSVLegacyTimeParserSuite] .exclude("SPARK-23786: warning should be printed if CSV header doesn't conform to schema") // file cars.csv include null string, Arrow not support to read diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 9b469a98d137..128e52a79b77 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -215,6 +215,15 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Gluten - test for FAILFAST parsing mode") // file cars.csv include null string, Arrow not support to read .exclude("old csv data source name works") + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch + // Early Filter and Projection Push-Down generated an invalid plan + .exclude("SPARK-26208: write and read empty data to csv file with headers") enableSuite[GlutenCSVLegacyTimeParserSuite] // file cars.csv include null string, Arrow not support to read .exclude("DDL test with schema") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 1afa203ab6f5..6ea29847b0a6 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -195,6 +195,15 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Gluten - test for FAILFAST parsing mode") // file cars.csv include null string, Arrow not support to read .exclude("old csv data source name works") + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") + // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch + // Early Filter and Projection Push-Down generated an invalid plan + .exclude("SPARK-26208: write and read empty data to csv file with headers") enableSuite[GlutenCSVLegacyTimeParserSuite] // file cars.csv include null string, Arrow not support to read .exclude("DDL test with schema") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 61353d99f7d1..e6e42acb31a2 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -196,8 +196,17 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] .exclude("Gluten - test for FAILFAST parsing mode") + // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch + // Early Filter and Projection Push-Down generated an invalid plan + .exclude("SPARK-26208: write and read empty data to csv file with headers") // file cars.csv include null string, Arrow not support to read .exclude("old csv data source name works") + .exclude("DDL test with schema") + .exclude("save csv") + .exclude("save csv with compression codec option") + .exclude("save csv with empty fields with user defined empty values") + .exclude("save csv with quote") + .exclude("SPARK-13543 Write the output as uncompressed via option()") enableSuite[GlutenCSVLegacyTimeParserSuite] // file cars.csv include null string, Arrow not support to read .exclude("DDL test with schema") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala index 38e6c9873ee0..cb7ce87f97da 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala @@ -113,6 +113,7 @@ class GlutenCSVv2Suite extends GlutenCSVSuite { override def sparkConf: SparkConf = super.sparkConf .set(SQLConf.USE_V1_SOURCE_LIST, "") + .set(GlutenConfig.NATIVE_ARROW_READER_ENABLED.key, "true") override def testNameBlackList: Seq[String] = Seq( // overwritten with different test diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 4db784782c1e..e445dd33a585 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -100,3 +100,7 @@ abstract class BatchScanExecShim( ) } } + +abstract class ArrowBatchScanExecShim(original: BatchScanExec) extends DataSourceV2ScanExecBase { + @transient override lazy val partitions: Seq[InputPartition] = original.partitions +} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 76556052c758..06eb69a35973 100644 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -137,3 +137,9 @@ abstract class BatchScanExecShim( Boolean.box(replicatePartitions)) } } + +abstract class ArrowBatchScanExecShim(original: BatchScanExec) extends DataSourceV2ScanExecBase { + @transient override lazy val inputPartitions: Seq[InputPartition] = original.inputPartitions + + override def keyGroupedPartitioning: Option[Seq[Expression]] = original.keyGroupedPartitioning +} diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index ca9a7eb2d071..64afc8193f4e 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -150,3 +150,11 @@ abstract class BatchScanExecShim( } } } + +abstract class ArrowBatchScanExecShim(original: BatchScanExec) extends DataSourceV2ScanExecBase { + @transient override lazy val inputPartitions: Seq[InputPartition] = original.inputPartitions + + override def keyGroupedPartitioning: Option[Seq[Expression]] = original.keyGroupedPartitioning + + override def ordering: Option[Seq[SortOrder]] = original.ordering +} diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 47adf16fb0e7..8949a46a1ddd 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -152,3 +152,11 @@ abstract class BatchScanExecShim( } } } + +abstract class ArrowBatchScanExecShim(original: BatchScanExec) extends DataSourceV2ScanExecBase { + @transient override lazy val inputPartitions: Seq[InputPartition] = original.inputPartitions + + override def keyGroupedPartitioning: Option[Seq[Expression]] = original.keyGroupedPartitioning + + override def ordering: Option[Seq[SortOrder]] = original.ordering +} From 0d4258d7848a9349aba5ec143c503407ba8f50be Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 16 May 2024 19:48:42 +0800 Subject: [PATCH 085/402] [VL] Daily Update Velox Version (2024_05_16) (#5756) --- .github/workflows/velox_docker_cache.yml | 78 ++++++++++++------------ ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index ec95f48a24f6..cbc24384d274 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -84,42 +84,42 @@ jobs: with: path: '${{ env.CCACHE_DIR }}' key: ccache-ubuntu-release-default - ccache-native-lib-centos-velox-ut: - runs-on: ubuntu-20.04 - env: - CCACHE_DIR: "${{ github.workspace }}/.ccache" - container: ghcr.io/facebookincubator/velox-dev:circleci-avx - steps: - - uses: actions/checkout@v2 - - name: Setup java and maven - run: | - yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ - wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - - name: Get Ccache - uses: actions/cache/restore@v3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-centos-release-default - - name: Ensure Cache Dirs Exists - working-directory: ${{ github.workspace }} - run: | - mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - rm -rf /opt/miniconda-for-velox/ - cd ep/build-velox/src && \ - ./get_velox.sh - cd ../build/velox_ep/ - source /opt/rh/gcc-toolset-9/enable - make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" - - - name: CCache after - run: | - ccache -s - - - uses: actions/cache/save@v3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-centos-release-default \ No newline at end of file +# ccache-native-lib-centos-velox-ut: +# runs-on: ubuntu-20.04 +# env: +# CCACHE_DIR: "${{ github.workspace }}/.ccache" +# container: ghcr.io/facebookincubator/velox-dev:circleci-avx +# steps: +# - uses: actions/checkout@v2 +# - name: Setup java and maven +# run: | +# yum install sudo patch java-1.8.0-openjdk-devel wget -y && \ +# wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz +# tar -xvf apache-maven-3.8.8-bin.tar.gz +# mv apache-maven-3.8.8 /usr/lib/maven +# - name: Get Ccache +# uses: actions/cache/restore@v3 +# with: +# path: '${{ env.CCACHE_DIR }}' +# key: ccache-centos-release-default +# - name: Ensure Cache Dirs Exists +# working-directory: ${{ github.workspace }} +# run: | +# mkdir -p '${{ env.CCACHE_DIR }}' +# - name: Build Gluten velox third party +# run: | +# rm -rf /opt/miniconda-for-velox/ +# cd ep/build-velox/src && \ +# ./get_velox.sh +# cd ../build/velox_ep/ +# source /opt/rh/gcc-toolset-9/enable +# make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" +# +# - name: CCache after +# run: | +# ccache -s +# +# - uses: actions/cache/save@v3 +# with: +# path: '${{ env.CCACHE_DIR }}' +# key: ccache-centos-release-default \ No newline at end of file diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 497befbe6018..17a0b3796f27 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_15 +VELOX_BRANCH=2024_05_16 VELOX_HOME="" #Set on run gluten on HDFS From c3599b304cbc1b613796763aa157d500e90817cf Mon Sep 17 00:00:00 2001 From: James Xu Date: Thu, 16 May 2024 22:46:25 +0800 Subject: [PATCH 086/402] [GLUTEN-5759][CORE] Optimze checkGlutenOperatorMatch to show clearer error message (#5760) --- .../apache/gluten/execution/WholeStageTransformerSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index c52002b68a27..bb1867d96f3c 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -242,8 +242,8 @@ abstract class WholeStageTransformerSuite val executedPlan = getExecutedPlan(df) assert( executedPlan.exists(plan => tag.runtimeClass.isInstance(plan)), - s"Expect ${tag.runtimeClass.getClass.getSimpleName} exists " + - s"in executedPlan:\n $executedPlan" + s"Expect ${tag.runtimeClass.getSimpleName} exists " + + s"in executedPlan:\n ${executedPlan.last}" ) } From 5e782a13d060aa87c6e5b5144af85a3a0361196e Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 17 May 2024 08:25:09 +0800 Subject: [PATCH 087/402] [VL][CI] Disable Velox UT (#5780) --- .github/workflows/velox_docker_cache.yml | 60 +++++++++---------- ...lox_ut.yml => velox_velox_ut.yml.disabled} | 0 2 files changed, 30 insertions(+), 30 deletions(-) rename .github/workflows/{velox_velox_ut.yml => velox_velox_ut.yml.disabled} (100%) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index cbc24384d274..51b0bae2f407 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -52,38 +52,38 @@ jobs: path: ./cpp/build/releases/ key: cache-velox-build-${{ hashFiles('./cache-key') }} - ccache-native-lib-ubuntu-velox-ut: - runs-on: ubuntu-20.04 - env: - CCACHE_DIR: "${{ github.workspace }}/.ccache" - container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx - steps: - - uses: actions/checkout@v2 - - name: Get Ccache - uses: actions/cache/restore@v3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-ubuntu-release-default - - name: Ensure Cache Dirs Exists - working-directory: ${{ github.workspace }} - run: | - mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - rm -rf /opt/miniconda-for-velox/ - cd ep/build-velox/src && \ - ./get_velox.sh - cd ../build/velox_ep/ - make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" + # ccache-native-lib-ubuntu-velox-ut: + # runs-on: ubuntu-20.04 + # env: + # CCACHE_DIR: "${{ github.workspace }}/.ccache" + # container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx + # steps: + # - uses: actions/checkout@v2 + # - name: Get Ccache + # uses: actions/cache/restore@v3 + # with: + # path: '${{ env.CCACHE_DIR }}' + # key: ccache-ubuntu-release-default + # - name: Ensure Cache Dirs Exists + # working-directory: ${{ github.workspace }} + # run: | + # mkdir -p '${{ env.CCACHE_DIR }}' + # - name: Build Gluten velox third party + # run: | + # rm -rf /opt/miniconda-for-velox/ + # cd ep/build-velox/src && \ + # ./get_velox.sh + # cd ../build/velox_ep/ + # make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" - - name: CCache after - run: | - ccache -vs + # - name: CCache after + # run: | + # ccache -vs - - uses: actions/cache/save@v3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-ubuntu-release-default + # - uses: actions/cache/save@v3 + # with: + # path: '${{ env.CCACHE_DIR }}' + # key: ccache-ubuntu-release-default # ccache-native-lib-centos-velox-ut: # runs-on: ubuntu-20.04 # env: diff --git a/.github/workflows/velox_velox_ut.yml b/.github/workflows/velox_velox_ut.yml.disabled similarity index 100% rename from .github/workflows/velox_velox_ut.yml rename to .github/workflows/velox_velox_ut.yml.disabled From 022c208564dbb6dc59b7d64fbc4ad002c166e012 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Fri, 17 May 2024 10:27:34 +0800 Subject: [PATCH 088/402] [VL] Refine CMAKE_CXX_FLAGS setting logic (#5769) --- cpp/CMakeLists.txt | 58 +++++++++++++++++++++++++--------------- cpp/core/CMakeLists.txt | 7 +++-- cpp/velox/CMakeLists.txt | 23 +++++++++++----- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7c38d8cc4492..28c28a5bd000 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -16,6 +16,9 @@ cmake_minimum_required(VERSION 3.16) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + # The set(CACHE) command does not remove any normal variable of the same name from the current scope # https://cmake.org/cmake/help/latest/policy/CMP0126.html if(POLICY CMP0126) @@ -82,46 +85,59 @@ else () message(STATUS "Add definition NDEBUG") endif() -add_compile_options(-Wall) -add_compile_options(-Wno-sign-compare) -add_compile_options(-Wno-comment) - -add_compile_options(-Werror) -add_compile_options(-Wno-error=parentheses) -add_compile_options(-Wno-error=unused-function) -add_compile_options(-Wno-error=unused-variable) -add_compile_options(-Wno-strict-aliasing) +set(KNOWN_WARNINGS + "-Wall \ + -Wno-sign-compare \ + -Wno-comment \ + -Werror \ + -Wno-error=parentheses \ + -Wno-error=unused-function \ + -Wno-error=unused-variable \ + -Wno-strict-aliasing \ + -Wno-ignored-qualifiers") if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - add_compile_options(-Wno-error=unused-but-set-variable) + set(KNOWN_WARNINGS + "-Wno-error=unused-but-set-variable \ + ${KNOWN_WARNINGS}") if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11) - add_compile_options(-Wno-error=maybe-uninitialized) + set(KNOWN_WARNINGS + "-Wno-error=maybe-uninitialized \ + ${KNOWN_WARNINGS}") endif() elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") # Experimental - add_compile_options(-Wno-implicit-int-float-conversion) - add_compile_options(-Wno-nullability-completeness) - add_compile_options(-Wno-mismatched-tags) + set(KNOWN_WARNINGS + "-Wno-implicit-int-float-conversion \ + -Wno-nullability-completeness \ + -Wno-mismatched-tags \ + ${KNOWN_WARNINGS}") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") # Experimental - add_compile_options(-Wno-implicit-int-float-conversion) - add_compile_options(-Wno-nullability-completeness) - add_compile_options(-Wno-mismatched-tags) + set(KNOWN_WARNINGS + "-Wno-implicit-int-float-conversion \ + -Wno-nullability-completeness \ + -Wno-mismatched-tags \ + ${KNOWN_WARNINGS}") else() message(FATAL_ERROR "Unsupported compiler ID: ${CMAKE_CXX_COMPILER_ID}") endif() # see https://issues.apache.org/jira/browse/ARROW-4665 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - add_compile_options(-Wno-macro-redefined) - add_compile_options(-Wno-nullability-completeness) - add_compile_options(-Wno-pessimizing-move) - add_compile_options(-Wno-mismatched-tags) + set(KNOWN_WARNINGS + "-Wno-macro-redefined \ + -Wno-nullability-completeness \ + -Wno-pessimizing-move \ + -Wno-mismatched-tags \ + ${KNOWN_WARNINGS}") # Specific definition for an issue with boost/stacktrace when building on macOS. # See https://github.com/boostorg/stacktrace/issues/88 and comments therein. add_compile_definitions(_GNU_SOURCE) endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${KNOWN_WARNINGS}") + # # Dependencies # diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index c369a5bc38eb..4a8ae0e47c4b 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -22,10 +22,6 @@ include(FindPkgConfig) include(GNUInstallDirs) include(CheckCXXCompilerFlag) -set(CMAKE_CXX_STANDARD 17) - -set(CMAKE_CXX_STANDARD_REQUIRED ON) - # Only set arch=native for non-AppleClang compilers. if (NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") @@ -173,6 +169,9 @@ set_source_files_properties(${GLUTEN_PROTO_OUTPUT_FILES} PROPERTIES GENERATED TR get_filename_component(GLUTEN_PROTO_DIR ${GLUTEN_PROTO_SRC_DIR}/ DIRECTORY) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes") + +message("Core module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") + set(SPARK_COLUMNAR_PLUGIN_SRCS ${SUBSTRAIT_PROTO_SRCS} ${GLUTEN_PROTO_SRCS} diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 9ad14fdf7f9d..9e5f08b1cb46 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -23,11 +23,6 @@ include(GNUInstallDirs) include(CheckCXXCompilerFlag) include(FindPackageHandleStandardArgs) -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -mavx2") -endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes") if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess") @@ -52,6 +47,20 @@ if (NOT DEFINED VELOX_HOME) message(STATUS "Set VELOX_HOME to ${VELOX_HOME}") endif() +# Keep same compile option with Velox. +execute_process( + COMMAND + bash -c + "( source ${VELOX_HOME}/scripts/setup-helper-functions.sh && echo -n $(get_cxx_flags $ENV{CPU_TARGET}))" + OUTPUT_VARIABLE SCRIPT_CXX_FLAGS + RESULT_VARIABLE COMMAND_STATUS) +if(COMMAND_STATUS EQUAL "1") + message(FATAL_ERROR "Unable to determine compiler flags!") +endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}") + +message("Velox module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") + # User can specify VELOX_BUILD_PATH, if Velox are built elsewhere. if(NOT DEFINED VELOX_BUILD_PATH) if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") @@ -125,7 +134,7 @@ macro(ADD_VELOX_DEPENDENCIES) add_velox_dependency(dwio::common::test::utils "${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a") endif() add_velox_dependency(exec "${VELOX_COMPONENTS_PATH}/exec/libvelox_exec.a") - + if(BUILD_TESTS) add_velox_dependency(parse::parser "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_parser.a") add_velox_dependency(duckdb::parser "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_parser.a") @@ -162,7 +171,7 @@ macro(ADD_VELOX_DEPENDENCIES) add_velox_dependency(parquet::reader::duckdb_conversion "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_conversion.a") add_duckdb() - + add_velox_dependency(tpch::gen "${VELOX_COMPONENTS_PATH}/tpch/gen/libvelox_tpch_gen.a") add_velox_dependency(dbgen "${VELOX_COMPONENTS_PATH}/tpch/gen/dbgen/libvelox_dbgen.a") endif() From db8496bdb16750b424d351c2fd23d831f4af769c Mon Sep 17 00:00:00 2001 From: Marcus Markiewicz <43656407+supermem613@users.noreply.github.com> Date: Thu, 16 May 2024 23:09:02 -0400 Subject: [PATCH 089/402] [GLUTEN-5438] feat: Dynamically sizing off-heap memory (#5439) ## What changes were proposed in this pull request? Today, in Spark we specify the on-heap and off-heap memory sizes as a configuration value read at the beginning of executing a job. With this change, we are exposing a new feature that is enabled with a new spark.gluten.memory.dynamic.offHeap.sizing.enabled setting. When this setting is configured to true, the offheap setting will be ignored in Gluten and we will size the offheap as the same size as the spark.executor.memory setting. We will then proceed to enforcing a total memory quota, calculated by the sum of what memory is committed and in use in the Java heap (calculated with Runtime.getRuntime().totalMemory() - Runtime.GetRuntime().freeMemory()) plus the tracked off-heap memory in TreeMemoryConsumer. When there is an allocation that would tide us over this total amount of committed memory, we will fail the allocation and trigger an OOM. Note that with this change, we perform the "quota check" when an allocation in the native engine is informed to Gluten. In practice, this means that it is possible that the Java codebase can oversubscribe memory as it allocates, which is under the on-heap quota, although there is enough off-heap usage where we should fail the allocation. A test exercising this setting is part of this change. Fixes: #5438 ## How was this patch tested? Manual testing with Spark and included test --- .../execution/DynamicOffHeapSizingTest.scala | 60 ++++++++++++ .../DynamicOffHeapSizingMemoryTarget.java | 95 +++++++++++++++++++ .../memory/memtarget/MemoryTargetVisitor.java | 2 + .../memory/memtarget/MemoryTargets.java | 11 ++- .../memtarget/ThrowOnOomMemoryTarget.java | 19 +++- .../org/apache/gluten/GlutenPlugin.scala | 78 ++++++++++++--- .../apache/spark/memory/SparkMemoryUtil.scala | 7 +- .../org/apache/gluten/GlutenConfig.scala | 36 +++++++ 8 files changed, 291 insertions(+), 17 deletions(-) create mode 100644 backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala create mode 100644 gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala new file mode 100644 index 000000000000..56fc6eac3e11 --- /dev/null +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.gluten.benchmarks.RandomParquetDataGenerator +import org.apache.gluten.tags.SkipTestTags + +import org.apache.spark.SparkConf + +@SkipTestTags +class DynamicOffHeapSizingTest extends VeloxWholeStageTransformerSuite { + override protected val resourcePath: String = "/tpch-data-parquet-velox" + override protected val fileFormat: String = "parquet" + + private val dataGenerator = RandomParquetDataGenerator(System.currentTimeMillis()) + private val outputPath = getClass.getResource("/").getPath + "dynamicoffheapsizing_output.parquet" + private val AGG_SQL = + """select f_1, count(DISTINCT f_1) + |from tbl group + |group by 1""".stripMargin + + override def beforeAll(): Unit = { + super.beforeAll() + } + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + .set("spark.executor.memory", "6GB") + .set("spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction", "0.8") + .set("spark.gluten.memory.dynamic.offHeap.sizing.enabled", "true") + } + + def getRootCause(e: Throwable): Throwable = { + if (e.getCause == null) { + return e + } + getRootCause(e.getCause) + } + + test("Dynamic Off-Heap Sizing") { + System.gc() + dataGenerator.generateRandomData(spark, Some(outputPath)) + spark.read.format("parquet").load(outputPath).createOrReplaceTempView("tbl") + spark.sql(AGG_SQL) + } +} diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java new file mode 100644 index 000000000000..b7f15d830bed --- /dev/null +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.memory.memtarget; + +import org.apache.gluten.GlutenConfig; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicLong; + +public class DynamicOffHeapSizingMemoryTarget implements MemoryTarget { + private static final Logger LOG = LoggerFactory.getLogger(DynamicOffHeapSizingMemoryTarget.class); + private final MemoryTarget delegated; + // When dynamic off-heap sizing is enabled, the off-heap should be sized for the total usable + // memory, so we can use it as the max memory we will use. + private static final long MAX_MEMORY_IN_BYTES = GlutenConfig.getConf().offHeapMemorySize(); + private static final AtomicLong USED_OFFHEAP_BYTES = new AtomicLong(); + + public DynamicOffHeapSizingMemoryTarget(MemoryTarget delegated) { + this.delegated = delegated; + } + + @Override + public long borrow(long size) { + if (size == 0) { + return 0; + } + + long totalMemory = Runtime.getRuntime().totalMemory(); + long freeMemory = Runtime.getRuntime().freeMemory(); + long usedOnHeapBytes = (totalMemory - freeMemory); + long usedOffHeapBytesNow = USED_OFFHEAP_BYTES.get(); + + if (size + usedOffHeapBytesNow + usedOnHeapBytes > MAX_MEMORY_IN_BYTES) { + LOG.warn( + String.format( + "Failing allocation as unified memory is OOM. " + + "Used Off-heap: %d, Used On-Heap: %d, " + + "Free On-heap: %d, Total On-heap: %d, " + + "Max On-heap: %d, Allocation: %d.", + usedOffHeapBytesNow, + usedOnHeapBytes, + freeMemory, + totalMemory, + MAX_MEMORY_IN_BYTES, + size)); + + return 0; + } + + long reserved = delegated.borrow(size); + + USED_OFFHEAP_BYTES.addAndGet(reserved); + + return reserved; + } + + @Override + public long repay(long size) { + long unreserved = delegated.repay(size); + + USED_OFFHEAP_BYTES.addAndGet(-unreserved); + + return unreserved; + } + + @Override + public long usedBytes() { + return delegated.usedBytes(); + } + + @Override + public T accept(MemoryTargetVisitor visitor) { + return visitor.visit(this); + } + + public MemoryTarget delegated() { + return delegated; + } +} diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java index caff2605d923..e58dbb295b08 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java @@ -33,4 +33,6 @@ public interface MemoryTargetVisitor { T visit(LoggingMemoryTarget loggingMemoryTarget); T visit(NoopMemoryTarget noopMemoryTarget); + + T visit(DynamicOffHeapSizingMemoryTarget dynamicOffHeapSizingMemoryTarget); } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java index ccb4beee8475..2d6fc0748464 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java @@ -43,6 +43,14 @@ public static MemoryTarget overAcquire( return new OverAcquire(target, overTarget, overAcquiredRatio); } + public static MemoryTarget dynamicOffHeapSizingIfEnabled(MemoryTarget memoryTarget) { + if (GlutenConfig.getConf().dynamicOffHeapSizingEnabled()) { + return new DynamicOffHeapSizingMemoryTarget(memoryTarget); + } + + return memoryTarget; + } + public static MemoryTarget newConsumer( TaskMemoryManager tmm, String name, @@ -54,6 +62,7 @@ public static MemoryTarget newConsumer( } else { factory = TreeMemoryConsumers.shared(); } - return factory.newConsumer(tmm, name, spillers, virtualChildren); + + return dynamicOffHeapSizingIfEnabled(factory.newConsumer(tmm, name, spillers, virtualChildren)); } } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java index 6621f3b1683f..e6b6ba07eb6b 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java @@ -52,8 +52,10 @@ public long borrow(long size) { .append( String.format( "Not enough spark off-heap execution memory. Acquired: %s, granted: %s. " - + "Try tweaking config option spark.memory.offHeap.size to get larger space " - + "to run this application. %n", + + "Try tweaking config option spark.memory.offHeap.size to get larger " + + "space to run this application " + + "(if spark.gluten.memory.dynamic.offHeap.sizing.enabled " + + "is not enabled). %n", Utils.bytesToString(size), Utils.bytesToString(granted))) .append("Current config settings: ") .append(System.lineSeparator()) @@ -83,6 +85,19 @@ public long borrow(long size) { .getConfString( GlutenConfig$.MODULE$ .GLUTEN_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES_KEY())))) + .append(System.lineSeparator()) + .append( + String.format( + "\t%s=%s", + GlutenConfig$.MODULE$.GLUTEN_OFFHEAP_ENABLED(), + SQLConf.get().getConfString(GlutenConfig$.MODULE$.GLUTEN_OFFHEAP_ENABLED()))) + .append(System.lineSeparator()) + .append( + String.format( + "\t%s=%s", + GlutenConfig$.MODULE$.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED(), + SQLConf.get() + .getConfString(GlutenConfig$.MODULE$.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED()))) .append(System.lineSeparator()); // Dump all consumer usages to exception body errorBuilder.append(SparkMemoryUtil.dumpMemoryTargetStats(target)); diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index adb3f418907f..6c3d62c1e207 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -148,9 +148,10 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { // check memory off-heap enabled and size val minOffHeapSize = "1MB" if ( - !conf.getBoolean(GlutenConfig.GLUTEN_OFFHEAP_ENABLED, false) || - conf.getSizeAsBytes(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY, 0) < JavaUtils.byteStringAsBytes( - minOffHeapSize) + !conf.getBoolean(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED, false) && + (!conf.getBoolean(GlutenConfig.GLUTEN_OFFHEAP_ENABLED, false) || + conf.getSizeAsBytes(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY, 0) < JavaUtils.byteStringAsBytes( + minOffHeapSize)) ) { throw new GlutenException( s"Must set '${GlutenConfig.GLUTEN_OFFHEAP_ENABLED}' to true " + @@ -164,20 +165,71 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { // task slots val taskSlots = SparkResourceUtil.getTaskSlots(conf) - // Optimistic off-heap sizes, assuming all storage memory can be borrowed into execution memory - // pool, regardless of Spark option spark.memory.storageFraction. - val offHeapSize = conf.getSizeAsBytes(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY) + var onHeapSize: Long = + if (conf.contains(GlutenConfig.GLUTEN_ONHEAP_SIZE_KEY)) { + conf.getSizeAsBytes(GlutenConfig.GLUTEN_ONHEAP_SIZE_KEY) + } else { + // 1GB default + 1024 * 1024 * 1024 + } + + // If dynamic off-heap sizing is enabled, the off-heap size is calculated based on the on-heap + // size. Otherwise, the off-heap size is set to the value specified by the user (if any). + // Note that this means that we will IGNORE the off-heap size specified by the user if the + // dynamic off-heap feature is enabled. + var offHeapSize: Long = + if (conf.getBoolean(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED, false)) { + // Since when dynamic off-heap sizing is enabled, we commingle on-heap + // and off-heap memory, we set the off-heap size to the usable on-heap size. We will + // size it with a memory fraction, which can be aggressively set, but the default + // is using the same way that Spark sizes on-heap memory: + // + // spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction * + // (spark.executor.memory - 300MB). + // + // We will be careful to use the same configuration settings as Spark to ensure + // that we are sizing the off-heap memory in the same way as Spark sizes on-heap memory. + // The 300MB value, unfortunately, is hard-coded in Spark code. + ((onHeapSize - (300 * 1024 * 1024)) * + conf.getDouble(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION, 0.6d)).toLong + } else if (conf.contains(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY)) { + // Optimistic off-heap sizes, assuming all storage memory can be borrowed into execution + // memory pool, regardless of Spark option spark.memory.storageFraction. + conf.getSizeAsBytes(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY) + } else { + // Default Spark Value. + 0L + } + conf.set(GlutenConfig.GLUTEN_OFFHEAP_SIZE_IN_BYTES_KEY, offHeapSize.toString) + conf.set(GlutenConfig.GLUTEN_OFFHEAP_SIZE_KEY, offHeapSize.toString) + val offHeapPerTask = offHeapSize / taskSlots conf.set(GlutenConfig.GLUTEN_TASK_OFFHEAP_SIZE_IN_BYTES_KEY, offHeapPerTask.toString) - // Pessimistic off-heap sizes, with the assumption that all non-borrowable storage memory - // determined by spark.memory.storageFraction was used. - val fraction = 1.0d - conf.getDouble("spark.memory.storageFraction", 0.5d) - val conservativeOffHeapPerTask = (offHeapSize * fraction).toLong / taskSlots - conf.set( - GlutenConfig.GLUTEN_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES_KEY, - conservativeOffHeapPerTask.toString) + // If we are using dynamic off-heap sizing, we should also enable off-heap memory + // officially. + if (conf.getBoolean(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED, false)) { + conf.set(GlutenConfig.GLUTEN_OFFHEAP_ENABLED, "true") + + // We already sized the off-heap per task in a conservative manner, so we can just + // use it. + conf.set( + GlutenConfig.GLUTEN_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES_KEY, + offHeapPerTask.toString) + } else { + // Let's make sure this is set to false explicitly if it is not on as it + // is looked up when throwing OOF exceptions. + conf.set(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED, "false") + + // Pessimistic off-heap sizes, with the assumption that all non-borrowable storage memory + // determined by spark.memory.storageFraction was used. + val fraction = 1.0d - conf.getDouble("spark.memory.storageFraction", 0.5d) + val conservativeOffHeapPerTask = (offHeapSize * fraction).toLong / taskSlots + conf.set( + GlutenConfig.GLUTEN_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES_KEY, + conservativeOffHeapPerTask.toString) + } // disable vanilla columnar readers, to prevent columnar-to-columnar conversions if (BackendsApiManager.getSettings.disableVanillaColumnarReaders(conf)) { diff --git a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala index 8bf88ef7d15c..48ed08fb71ce 100644 --- a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.memory -import org.apache.gluten.memory.memtarget.{KnownNameAndStats, LoggingMemoryTarget, MemoryTarget, MemoryTargetVisitor, NoopMemoryTarget, OverAcquire, ThrowOnOomMemoryTarget, TreeMemoryTargets} +import org.apache.gluten.memory.memtarget.{DynamicOffHeapSizingMemoryTarget, KnownNameAndStats, LoggingMemoryTarget, MemoryTarget, MemoryTargetVisitor, NoopMemoryTarget, OverAcquire, ThrowOnOomMemoryTarget, TreeMemoryTargets} import org.apache.gluten.memory.memtarget.spark.{RegularMemoryConsumer, TreeMemoryConsumer} import org.apache.gluten.proto.MemoryUsageStats @@ -117,6 +117,11 @@ object SparkMemoryUtil { override def visit(noopMemoryTarget: NoopMemoryTarget): KnownNameAndStats = { noopMemoryTarget } + + override def visit(dynamicOffHeapSizingMemoryTarget: DynamicOffHeapSizingMemoryTarget) + : KnownNameAndStats = { + dynamicOffHeapSizingMemoryTarget.delegated().accept(this) + } }) } diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 7a501e02deb0..ca8a9dce12c5 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -395,6 +395,9 @@ class GlutenConfig(conf: SQLConf) extends Logging { def awsSdkLogLevel: String = conf.getConf(AWS_SDK_LOG_LEVEL) def enableCastAvgAggregateFunction: Boolean = conf.getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED) + + def dynamicOffHeapSizingEnabled: Boolean = + conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) } object GlutenConfig { @@ -466,6 +469,7 @@ object GlutenConfig { val GLUTEN_CONFIG_PREFIX = "spark.gluten.sql.columnar.backend." // Private Spark configs. + val GLUTEN_ONHEAP_SIZE_KEY = "spark.executor.memory" val GLUTEN_OFFHEAP_SIZE_KEY = "spark.memory.offHeap.size" val GLUTEN_OFFHEAP_ENABLED = "spark.memory.offHeap.enabled" @@ -543,6 +547,10 @@ object GlutenConfig { val GLUTEN_UI_ENABLED = "spark.gluten.ui.enabled" + val GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED = "spark.gluten.memory.dynamic.offHeap.sizing.enabled" + val GLUTEN_DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION = + "spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction" + var ins: GlutenConfig = _ def getConf: GlutenConfig = { @@ -1835,4 +1843,32 @@ object GlutenConfig { .internal() .booleanConf .createWithDefault(true) + + val DYNAMIC_OFFHEAP_SIZING_ENABLED = + buildConf(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED) + .internal() + .doc( + "Experimental: When set to true, the offheap config (spark.memory.offHeap.size) will " + + "be ignored and instead we will consider onheap and offheap memory in combination, " + + "both counting towards the executor memory config (spark.executor.memory). We will " + + "make use of JVM APIs to determine how much onheap memory is use, alongside tracking " + + "offheap allocations made by Gluten. We will then proceed to enforcing a total memory " + + "quota, calculated by the sum of what memory is committed and in use in the Java " + + "heap. Since the calculation of the total quota happens as offheap allocation happens " + + "and not as JVM heap memory is allocated, it is possible that we can oversubscribe " + + "memory. Additionally, note that this change is experimental and may have performance " + + "implications.") + .booleanConf + .createWithDefault(false) + + val DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION = + buildConf(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION) + .internal() + .doc( + "Experimental: Determines the memory fraction used to determine the total " + + "memory available for offheap and onheap allocations when the dynamic offheap " + + "sizing feature is enabled. The default is set to match spark.executor.memoryFraction.") + .doubleConf + .checkValue(v => v >= 0 && v <= 1, "offheap sizing memory fraction must between [0, 1]") + .createWithDefault(0.6) } From 2f7c9985316540099897f35a3aa682f48a2b126e Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 17 May 2024 12:06:32 +0800 Subject: [PATCH 090/402] [GLUTEN-5775][CELEBORN] Fix invoke celebornShuffleId exception (#5776) --- .../shuffle/gluten/celeborn/CelebornShuffleManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index 15b109a65a38..a1a41f973249 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -259,7 +259,7 @@ public ShuffleWriter getWriter( } @SuppressWarnings("unchecked") CelebornShuffleHandle h = ((CelebornShuffleHandle) handle); - ShuffleClient client = + shuffleClient = CelebornUtils.getShuffleClient( h.appUniqueId(), h.lifecycleManagerHost(), @@ -279,7 +279,7 @@ public ShuffleWriter getWriter( ShuffleClient.class, CelebornShuffleHandle.class, TaskContext.class, - boolean.class); + Boolean.class); shuffleId = (int) celebornShuffleIdMethod.invoke(null, shuffleClient, h, context, true); Method trackMethod = @@ -298,7 +298,7 @@ public ShuffleWriter getWriter( if (h.dependency() instanceof ColumnarShuffleDependency) { // columnar-based shuffle return writerFactory.createShuffleWriterInstance( - shuffleId, h, context, celebornConf, client, metrics); + shuffleId, h, context, celebornConf, shuffleClient, metrics); } else { // row-based shuffle return vanillaCelebornShuffleManager().getWriter(handle, mapId, context, metrics); From 2087b57293d455c50c3b067108117bf5437573a4 Mon Sep 17 00:00:00 2001 From: James Xu Date: Fri, 17 May 2024 13:31:10 +0800 Subject: [PATCH 091/402] [GLUTEN-5777][VL] Supporting specify spark version when build (#5778) Support a new option: --spark_version for buildbundle-veloxbe.sh and builddeps-veloxbe.sh. By default the version is ALL which keeps the same behaviour as now. --- dev/buildbundle-veloxbe.sh | 19 ++++++++++--- dev/builddeps-veloxbe.sh | 14 ++++++++++ docs/get-started/build-guide.md | 49 +++++++++++++++++---------------- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh index 10c8e61e2424..eaa82730bb25 100755 --- a/dev/buildbundle-veloxbe.sh +++ b/dev/buildbundle-veloxbe.sh @@ -3,8 +3,19 @@ BASEDIR=$(dirname $0) source "$BASEDIR/builddeps-veloxbe.sh" +function build_for_spark { + spark_version=$1 + mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-$spark_version -DskipTests +} + cd $GLUTEN_DIR -mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests -mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests -mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests -mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests + +# SPARK_VERSION is defined in builddeps-veloxbe.sh +if [ "$SPARK_VERSION" = "ALL" ]; then + for spark_version in 3.2 3.3 3.4 3.5 + do + build_for_spark $spark_version + done +else + build_for_spark $SPARK_VERSION +fi diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 232c36d4053c..35976d37a036 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -32,6 +32,7 @@ VELOX_BRANCH="" VELOX_HOME="" VELOX_PARAMETER="" COMPILE_ARROW_JAVA=ON +SPARK_VERSION=ALL # set default number of threads as cpu cores minus 2 if [[ "$(uname)" == "Darwin" ]]; then @@ -142,6 +143,10 @@ do --num_threads=*) NUM_THREADS=("${arg#*=}") shift # Remove argument name from processing + ;; + --spark_version=*) + SPARK_VERSION=("${arg#*=}") + shift # Remove argument name from processing ;; *) OTHER_ARGUMENTS+=("$1") @@ -173,6 +178,15 @@ if [ "$ENABLE_VCPKG" = "ON" ]; then eval "$envs" fi +if [ "$SPARK_VERSION" = "3.2" ] || [ "$SPARK_VERSION" = "3.3" ] \ + || [ "$SPARK_VERSION" = "3.4" ] || [ "$SPARK_VERSION" = "3.5" ] \ + || [ "$SPARK_VERSION" = "ALL" ]; then + echo "Building for Spark $SPARK_VERSION" +else + echo "Invalid Spark version: $SPARK_VERSION" + exit 1 +fi + concat_velox_param function build_velox { diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index 90c2ed119914..3db2244ba229 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -8,30 +8,31 @@ parent: Getting-Started ### Native build parameters for buildbundle-veloxbe.sh or builddeps-veloxbe.sh Please set them via `--`, e.g. `--build_type=Release`. -| Parameters | Description | Default | -|------------------------|----------------------------------------------------------------------------|---------| -| build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | -| build_tests | Build gluten cpp tests. | OFF | -| build_examples | Build udf example. | OFF | -| build_benchmarks | Build gluten cpp benchmarks. | OFF | -| build_jemalloc | Build with jemalloc. | ON | -| build_protobuf | Build protobuf lib. | ON | -| enable_qat | Enable QAT for shuffle data de/compression. | OFF | -| enable_iaa | Enable IAA for shuffle data de/compression. | OFF | -| enable_hbm | Enable HBM allocator. | OFF | -| enable_s3 | Build with S3 support. | OFF | -| enable_gcs | Build with GCs support. | OFF | -| enable_hdfs | Build with HDFS support. | OFF | -| enable_abfs | Build with ABFS support. | OFF | -| enable_ep_cache | Enable caching for external project build (Velox). | OFF | -| enable_vcpkg | Enable vcpkg for static build. | OFF | -| run_setup_script | Run setup script to install Velox dependencies. | ON | -| velox_repo | Specify your own Velox repo to build. | "" | -| velox_branch | Specify your own Velox branch to build. | "" | -| velox_home | Specify your own Velox source path to build. | "" | -| build_velox_tests | Build Velox tests. | OFF | -| build_velox_benchmarks | Build Velox benchmarks (velox_tests and connectors will be disabled if ON) | OFF | -| compile_arrow_java | Compile arrow java for gluten build to use to fix invalid pointer issues. | ON | +| Parameters | Description | Default | +|------------------------|----------------------------------------------------------------------------------------------------|---------| +| build_type | Build type for Velox & gluten cpp, CMAKE_BUILD_TYPE. | Release | +| build_tests | Build gluten cpp tests. | OFF | +| build_examples | Build udf example. | OFF | +| build_benchmarks | Build gluten cpp benchmarks. | OFF | +| build_jemalloc | Build with jemalloc. | ON | +| build_protobuf | Build protobuf lib. | ON | +| enable_qat | Enable QAT for shuffle data de/compression. | OFF | +| enable_iaa | Enable IAA for shuffle data de/compression. | OFF | +| enable_hbm | Enable HBM allocator. | OFF | +| enable_s3 | Build with S3 support. | OFF | +| enable_gcs | Build with GCs support. | OFF | +| enable_hdfs | Build with HDFS support. | OFF | +| enable_abfs | Build with ABFS support. | OFF | +| enable_ep_cache | Enable caching for external project build (Velox). | OFF | +| enable_vcpkg | Enable vcpkg for static build. | OFF | +| run_setup_script | Run setup script to install Velox dependencies. | ON | +| velox_repo | Specify your own Velox repo to build. | "" | +| velox_branch | Specify your own Velox branch to build. | "" | +| velox_home | Specify your own Velox source path to build. | "" | +| build_velox_tests | Build Velox tests. | OFF | +| build_velox_benchmarks | Build Velox benchmarks (velox_tests and connectors will be disabled if ON) | OFF | +| compile_arrow_java | Compile arrow java for gluten build to use to fix invalid pointer issues. | ON | +| spark_version | Build for specified version of Spark(3.2, 3.3, 3.4, 3.5, ALL). `ALL` means build for all versions. | ALL | ### Velox build parameters for build_velox.sh Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`. From dfac04f6e9f471a67248a2b9a2e582c9d6f22597 Mon Sep 17 00:00:00 2001 From: Kerwin Zhang Date: Fri, 17 May 2024 13:44:54 +0800 Subject: [PATCH 092/402] [VL] Support celeborn sort based shuffle (#5675) --- cpp/core/jni/JniCommon.h | 18 +- cpp/core/jni/JniWrapper.cc | 30 +- cpp/core/shuffle/FallbackRangePartitioner.cc | 18 + cpp/core/shuffle/FallbackRangePartitioner.h | 6 + cpp/core/shuffle/HashPartitioner.cc | 49 ++- cpp/core/shuffle/HashPartitioner.h | 6 + cpp/core/shuffle/LocalPartitionWriter.cc | 4 + cpp/core/shuffle/LocalPartitionWriter.h | 2 + cpp/core/shuffle/Options.h | 16 +- cpp/core/shuffle/PartitionWriter.h | 6 + cpp/core/shuffle/Partitioner.h | 9 + cpp/core/shuffle/RoundRobinPartitioner.cc | 16 + cpp/core/shuffle/RoundRobinPartitioner.h | 6 + cpp/core/shuffle/ShuffleReader.cc | 4 + cpp/core/shuffle/ShuffleReader.h | 6 +- cpp/core/shuffle/ShuffleWriter.h | 12 +- cpp/core/shuffle/SinglePartitioner.cc | 9 + cpp/core/shuffle/SinglePartitioner.h | 6 + cpp/core/shuffle/rss/RssClient.h | 2 +- cpp/core/shuffle/rss/RssPartitionWriter.cc | 7 + cpp/core/shuffle/rss/RssPartitionWriter.h | 2 + cpp/velox/CMakeLists.txt | 3 +- cpp/velox/benchmarks/GenericBenchmark.cc | 5 +- cpp/velox/benchmarks/ShuffleSplitBenchmark.cc | 9 +- cpp/velox/compute/VeloxRuntime.cc | 33 +- ...iter.cc => VeloxHashBasedShuffleWriter.cc} | 161 +++---- .../shuffle/VeloxHashBasedShuffleWriter.h | 406 ++++++++++++++++++ cpp/velox/shuffle/VeloxShuffleReader.cc | 115 ++++- cpp/velox/shuffle/VeloxShuffleReader.h | 51 ++- cpp/velox/shuffle/VeloxShuffleWriter.h | 405 +++-------------- .../shuffle/VeloxSortBasedShuffleWriter.cc | 317 ++++++++++++++ .../shuffle/VeloxSortBasedShuffleWriter.h | 117 +++++ cpp/velox/tests/VeloxShuffleWriterTest.cc | 24 +- cpp/velox/utils/tests/LocalRssClient.h | 2 +- .../utils/tests/VeloxShuffleWriterTestBase.h | 102 +++-- .../celeborn/CelebornShuffleManager.java | 5 - ...lebornHashBasedColumnarShuffleWriter.scala | 8 + ...VeloxCelebornColumnarBatchSerializer.scala | 6 +- ...lebornHashBasedColumnarShuffleWriter.scala | 10 +- .../vectorized/ShuffleReaderJniWrapper.java | 3 +- .../vectorized/ShuffleWriterJniWrapper.java | 16 +- .../vectorized/ColumnarBatchSerializer.scala | 4 +- .../spark/shuffle/ColumnarShuffleWriter.scala | 2 +- .../VeloxUniffleColumnarShuffleWriter.java | 6 +- .../org/apache/gluten/GlutenConfig.scala | 5 + 45 files changed, 1501 insertions(+), 548 deletions(-) rename cpp/velox/shuffle/{VeloxShuffleWriter.cc => VeloxHashBasedShuffleWriter.cc} (90%) create mode 100644 cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h create mode 100644 cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc create mode 100644 cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index 29c38689c67e..aa3b2b8840c6 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -280,6 +280,20 @@ static inline arrow::Compression::type getCompressionType(JNIEnv* env, jstring c return compressionType; } +static inline const std::string getCompressionTypeStr(JNIEnv* env, jstring codecJstr) { + if (codecJstr == NULL) { + return "none"; + } + auto codec = env->GetStringUTFChars(codecJstr, JNI_FALSE); + + // Convert codec string into lowercase. + std::string codecLower; + std::transform(codec, codec + std::strlen(codec), std::back_inserter(codecLower), ::tolower); + + env->ReleaseStringUTFChars(codecJstr, codec); + return codecLower; +} + static inline gluten::CodecBackend getCodecBackend(JNIEnv* env, jstring codecJstr) { if (codecJstr == nullptr) { return gluten::CodecBackend::NONE; @@ -444,7 +458,7 @@ class JavaRssClient : public RssClient { env->DeleteGlobalRef(array_); } - int32_t pushPartitionData(int32_t partitionId, char* bytes, int64_t size) override { + int32_t pushPartitionData(int32_t partitionId, const char* bytes, int64_t size) override { JNIEnv* env; if (vm_->GetEnv(reinterpret_cast(&env), jniVersion) != JNI_OK) { throw gluten::GlutenException("JNIEnv was not attached to current thread"); @@ -457,7 +471,7 @@ class JavaRssClient : public RssClient { array_ = env->NewByteArray(size); array_ = static_cast(env->NewGlobalRef(array_)); } - env->SetByteArrayRegion(array_, 0, size, reinterpret_cast(bytes)); + env->SetByteArrayRegion(array_, 0, size, (jbyte*)bytes); jint javaBytesSize = env->CallIntMethod(javaRssShuffleWriter_, javaPushPartitionData_, partitionId, array_, size); checkException(env); return static_cast(javaBytesSize); diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 7363a9da0610..e70a017e07d0 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -831,8 +831,10 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe jlong taskAttemptId, jint startPartitionId, jint pushBufferMaxSize, + jlong sortBufferMaxSize, jobject partitionPusher, - jstring partitionWriterTypeJstr) { + jstring partitionWriterTypeJstr, + jstring shuffleWriterTypeJstr) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); auto memoryManager = jniCastOrThrow(memoryManagerHandle); @@ -866,10 +868,12 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe .mergeThreshold = mergeThreshold, .compressionThreshold = compressionThreshold, .compressionType = getCompressionType(env, codecJstr), + .compressionTypeStr = getCompressionTypeStr(env, codecJstr), .compressionLevel = compressionLevel, .bufferedWrite = true, .numSubDirs = numSubDirs, - .pushBufferMaxSize = pushBufferMaxSize > 0 ? pushBufferMaxSize : kDefaultShuffleWriterBufferSize}; + .pushBufferMaxSize = pushBufferMaxSize > 0 ? pushBufferMaxSize : kDefaultPushMemoryThreshold, + .sortBufferMaxSize = sortBufferMaxSize > 0 ? sortBufferMaxSize : kDefaultSortBufferThreshold}; if (codecJstr != NULL) { partitionWriterOptions.codecBackend = getCodecBackend(env, codecBackendJstr); partitionWriterOptions.compressionMode = getCompressionMode(env, compressionModeJstr); @@ -879,6 +883,15 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe auto partitionWriterTypeC = env->GetStringUTFChars(partitionWriterTypeJstr, JNI_FALSE); auto partitionWriterType = std::string(partitionWriterTypeC); env->ReleaseStringUTFChars(partitionWriterTypeJstr, partitionWriterTypeC); + + auto shuffleWriterTypeC = env->GetStringUTFChars(shuffleWriterTypeJstr, JNI_FALSE); + auto shuffleWriterType = std::string(shuffleWriterTypeC); + env->ReleaseStringUTFChars(shuffleWriterTypeJstr, shuffleWriterTypeC); + + if (shuffleWriterType == "sort") { + shuffleWriterOptions.shuffleWriterType = kSortShuffle; + } + if (partitionWriterType == "local") { if (dataFileJstr == NULL) { throw gluten::GlutenException(std::string("Shuffle DataFile can't be null")); @@ -962,7 +975,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe JNI_METHOD_END(kInvalidResourceHandle) } -JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_split( // NOLINT +JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_write( // NOLINT JNIEnv* env, jobject wrapper, jlong shuffleWriterHandle, @@ -981,7 +994,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe // The column batch maybe VeloxColumnBatch or ArrowCStructColumnarBatch(FallbackRangeShuffleWriter) auto batch = ctx->objectStore()->retrieve(batchHandle); auto numBytes = batch->numBytes(); - gluten::arrowAssertOkOrThrow(shuffleWriter->split(batch, memLimit), "Native split: shuffle writer split failed"); + gluten::arrowAssertOkOrThrow(shuffleWriter->write(batch, memLimit), "Native write: shuffle writer failed"); return numBytes; JNI_METHOD_END(kInvalidResourceHandle) } @@ -1058,7 +1071,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe jlong memoryManagerHandle, jstring compressionType, jstring compressionBackend, - jint batchSize) { + jint batchSize, + jstring shuffleWriterType) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); auto memoryManager = jniCastOrThrow(memoryManagerHandle); @@ -1066,11 +1080,16 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe auto pool = memoryManager->getArrowMemoryPool(); ShuffleReaderOptions options = ShuffleReaderOptions{}; options.compressionType = getCompressionType(env, compressionType); + options.compressionTypeStr = getCompressionTypeStr(env, compressionType); if (compressionType != nullptr) { options.codecBackend = getCodecBackend(env, compressionBackend); } options.batchSize = batchSize; // TODO: Add coalesce option and maximum coalesced size. + + if (jStringToCString(env, shuffleWriterType) == "sort") { + options.shuffleWriterType = kSortShuffle; + } std::shared_ptr schema = gluten::arrowGetOrThrow(arrow::ImportSchema(reinterpret_cast(cSchema))); @@ -1085,7 +1104,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe jobject jniIn) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto reader = ctx->objectStore()->retrieve(shuffleReaderHandle); std::shared_ptr in = std::make_shared(env, reader->getPool(), jniIn); auto outItr = reader->readStream(in); diff --git a/cpp/core/shuffle/FallbackRangePartitioner.cc b/cpp/core/shuffle/FallbackRangePartitioner.cc index 4bad50b5123c..677fcd114628 100644 --- a/cpp/core/shuffle/FallbackRangePartitioner.cc +++ b/cpp/core/shuffle/FallbackRangePartitioner.cc @@ -39,4 +39,22 @@ arrow::Status gluten::FallbackRangePartitioner::compute( return arrow::Status::OK(); } +arrow::Status gluten::FallbackRangePartitioner::compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) { + auto index = static_cast(vectorIndex) << 32; + for (auto i = 0; i < numRows; ++i) { + auto pid = pidArr[i]; + int64_t combined = index | (i & 0xFFFFFFFFLL); + auto& vec = rowVectorIndexMap[pid]; + vec.push_back(combined); + if (pid >= numPartitions_) { + return arrow::Status::Invalid( + "Partition id ", std::to_string(pid), " is equal or greater than ", std::to_string(numPartitions_)); + } + } + return arrow::Status::OK(); +} } // namespace gluten diff --git a/cpp/core/shuffle/FallbackRangePartitioner.h b/cpp/core/shuffle/FallbackRangePartitioner.h index f54dd1abc8c4..b06ce7e17251 100644 --- a/cpp/core/shuffle/FallbackRangePartitioner.h +++ b/cpp/core/shuffle/FallbackRangePartitioner.h @@ -30,6 +30,12 @@ class FallbackRangePartitioner final : public Partitioner { const int64_t numRows, std::vector& row2partition, std::vector& partition2RowCount) override; + + arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) override; }; } // namespace gluten diff --git a/cpp/core/shuffle/HashPartitioner.cc b/cpp/core/shuffle/HashPartitioner.cc index c62e3185f029..4a26dc67b2cb 100644 --- a/cpp/core/shuffle/HashPartitioner.cc +++ b/cpp/core/shuffle/HashPartitioner.cc @@ -19,6 +19,24 @@ namespace gluten { +int32_t computePid(const int32_t* pidArr, int64_t i, int32_t numPartitions) { + auto pid = pidArr[i] % numPartitions; +#if defined(__x86_64__) + // force to generate ASM + __asm__( + "lea (%[num_partitions],%[pid],1),%[tmp]\n" + "test %[pid],%[pid]\n" + "cmovs %[tmp],%[pid]\n" + : [pid] "+r"(pid) + : [num_partitions] "r"(numPartitions), [tmp] "r"(0)); +#else + if (pid < 0) { + pid += numPartitions_; + } +#endif + return pid; +} + arrow::Status gluten::HashPartitioner::compute( const int32_t* pidArr, const int64_t numRows, @@ -28,20 +46,7 @@ arrow::Status gluten::HashPartitioner::compute( std::fill(std::begin(partition2RowCount), std::end(partition2RowCount), 0); for (auto i = 0; i < numRows; ++i) { - auto pid = pidArr[i] % numPartitions_; -#if defined(__x86_64__) - // force to generate ASM - __asm__( - "lea (%[num_partitions],%[pid],1),%[tmp]\n" - "test %[pid],%[pid]\n" - "cmovs %[tmp],%[pid]\n" - : [pid] "+r"(pid) - : [num_partitions] "r"(numPartitions_), [tmp] "r"(0)); -#else - if (pid < 0) { - pid += numPartitions_; - } -#endif + auto pid = computePid(pidArr, i, numPartitions_); row2partition[i] = pid; } @@ -52,4 +57,20 @@ arrow::Status gluten::HashPartitioner::compute( return arrow::Status::OK(); } +arrow::Status gluten::HashPartitioner::compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) { + auto index = static_cast(vectorIndex) << 32; + for (auto i = 0; i < numRows; ++i) { + auto pid = computePid(pidArr, i, numPartitions_); + int64_t combined = index | (i & 0xFFFFFFFFLL); + auto& vec = rowVectorIndexMap[pid]; + vec.push_back(combined); + } + + return arrow::Status::OK(); +} + } // namespace gluten diff --git a/cpp/core/shuffle/HashPartitioner.h b/cpp/core/shuffle/HashPartitioner.h index fff01f939ddf..6cd664634983 100644 --- a/cpp/core/shuffle/HashPartitioner.h +++ b/cpp/core/shuffle/HashPartitioner.h @@ -30,6 +30,12 @@ class HashPartitioner final : public Partitioner { const int64_t numRows, std::vector& row2partition, std::vector& partition2RowCount) override; + + arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) override; }; } // namespace gluten diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc b/cpp/core/shuffle/LocalPartitionWriter.cc index 0582ce0e59f6..2fa0b954fa5f 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.cc +++ b/cpp/core/shuffle/LocalPartitionWriter.cc @@ -541,6 +541,10 @@ arrow::Status LocalPartitionWriter::evict( return arrow::Status::OK(); } +arrow::Status LocalPartitionWriter::evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) { + return arrow::Status::NotImplemented("Invalid code path for local shuffle writer: sort based is not supported."); +} + arrow::Status LocalPartitionWriter::reclaimFixedSize(int64_t size, int64_t* actual) { // Finish last spiller. RETURN_NOT_OK(finishSpill()); diff --git a/cpp/core/shuffle/LocalPartitionWriter.h b/cpp/core/shuffle/LocalPartitionWriter.h index 2cf4f2fd9a61..c2bfacd4b63e 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.h +++ b/cpp/core/shuffle/LocalPartitionWriter.h @@ -42,6 +42,8 @@ class LocalPartitionWriter : public PartitionWriter { bool reuseBuffers, bool hasComplexType) override; + arrow::Status evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) override; + /// The stop function performs several tasks: /// 1. Opens the final data file. /// 2. Iterates over each partition ID (pid) to: diff --git a/cpp/core/shuffle/Options.h b/cpp/core/shuffle/Options.h index d8fe1c802bff..4317ed6318ed 100644 --- a/cpp/core/shuffle/Options.h +++ b/cpp/core/shuffle/Options.h @@ -25,18 +25,24 @@ namespace gluten { static constexpr int16_t kDefaultBatchSize = 4096; -static constexpr int16_t kDefaultShuffleWriterBufferSize = 4096; +static constexpr int32_t kDefaultShuffleWriterBufferSize = 4096; +static constexpr int64_t kDefaultSortBufferThreshold = 64000000000; +static constexpr int64_t kDefaultPushMemoryThreshold = 4096; static constexpr int32_t kDefaultNumSubDirs = 64; static constexpr int32_t kDefaultCompressionThreshold = 100; +static const std::string kDefaultCompressionTypeStr = "lz4"; static constexpr int32_t kDefaultBufferAlignment = 64; static constexpr double kDefaultBufferReallocThreshold = 0.25; static constexpr double kDefaultMergeBufferThreshold = 0.25; static constexpr bool kEnableBufferedWrite = true; +enum ShuffleWriterType { kHashShuffle, kSortShuffle }; enum PartitionWriterType { kLocal, kRss }; struct ShuffleReaderOptions { arrow::Compression::type compressionType = arrow::Compression::type::LZ4_FRAME; + std::string compressionTypeStr = "lz4"; + ShuffleWriterType shuffleWriterType = kHashShuffle; CodecBackend codecBackend = CodecBackend::NONE; int32_t batchSize = kDefaultBatchSize; }; @@ -44,18 +50,20 @@ struct ShuffleReaderOptions { struct ShuffleWriterOptions { int32_t bufferSize = kDefaultShuffleWriterBufferSize; double bufferReallocThreshold = kDefaultBufferReallocThreshold; + int64_t pushMemoryThreshold = kDefaultPushMemoryThreshold; Partitioning partitioning = Partitioning::kRoundRobin; int64_t taskAttemptId = -1; int32_t startPartitionId = 0; int64_t threadId = -1; + ShuffleWriterType shuffleWriterType = kHashShuffle; }; struct PartitionWriterOptions { int32_t mergeBufferSize = kDefaultShuffleWriterBufferSize; double mergeThreshold = kDefaultMergeBufferThreshold; - int32_t compressionThreshold = kDefaultCompressionThreshold; arrow::Compression::type compressionType = arrow::Compression::LZ4_FRAME; + std::string compressionTypeStr = kDefaultCompressionTypeStr; CodecBackend codecBackend = CodecBackend::NONE; int32_t compressionLevel = arrow::util::kUseDefaultCompressionLevel; CompressionMode compressionMode = CompressionMode::BUFFER; @@ -64,7 +72,9 @@ struct PartitionWriterOptions { int32_t numSubDirs = kDefaultNumSubDirs; - int32_t pushBufferMaxSize = kDefaultShuffleWriterBufferSize; + int64_t pushBufferMaxSize = kDefaultPushMemoryThreshold; + + int64_t sortBufferMaxSize = kDefaultSortBufferThreshold; }; struct ShuffleWriterMetrics { diff --git a/cpp/core/shuffle/PartitionWriter.h b/cpp/core/shuffle/PartitionWriter.h index 42e97cf064da..93a6d04fe7cc 100644 --- a/cpp/core/shuffle/PartitionWriter.h +++ b/cpp/core/shuffle/PartitionWriter.h @@ -49,10 +49,16 @@ class PartitionWriter : public Reclaimable { bool reuseBuffers, bool hasComplexType) = 0; + virtual arrow::Status evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) = 0; + uint64_t cachedPayloadSize() { return payloadPool_->bytes_allocated(); } + PartitionWriterOptions& options() { + return options_; + } + protected: uint32_t numPartitions_; PartitionWriterOptions options_; diff --git a/cpp/core/shuffle/Partitioner.h b/cpp/core/shuffle/Partitioner.h index 8331b8a91797..b233f5b82673 100644 --- a/cpp/core/shuffle/Partitioner.h +++ b/cpp/core/shuffle/Partitioner.h @@ -18,7 +18,10 @@ #pragma once #include +#include + #include +#include #include #include "shuffle/Partitioning.h" @@ -40,6 +43,12 @@ class Partitioner { std::vector& row2partition, std::vector& partition2RowCount) = 0; + virtual arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) = 0; + protected: Partitioner(int32_t numPartitions, bool hasPid) : numPartitions_(numPartitions), hasPid_(hasPid) {} diff --git a/cpp/core/shuffle/RoundRobinPartitioner.cc b/cpp/core/shuffle/RoundRobinPartitioner.cc index b00680a18243..196f9308dde0 100644 --- a/cpp/core/shuffle/RoundRobinPartitioner.cc +++ b/cpp/core/shuffle/RoundRobinPartitioner.cc @@ -39,4 +39,20 @@ arrow::Status gluten::RoundRobinPartitioner::compute( return arrow::Status::OK(); } +arrow::Status gluten::RoundRobinPartitioner::compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) { + auto index = static_cast(vectorIndex) << 32; + for (int32_t i = 0; i < numRows; ++i) { + int64_t combined = index | (i & 0xFFFFFFFFLL); + auto& vec = rowVectorIndexMap[pidSelection_]; + vec.push_back(combined); + pidSelection_ = (pidSelection_ + 1) % numPartitions_; + } + + return arrow::Status::OK(); +} + } // namespace gluten diff --git a/cpp/core/shuffle/RoundRobinPartitioner.h b/cpp/core/shuffle/RoundRobinPartitioner.h index 5afd2832a82b..126a08eb9b29 100644 --- a/cpp/core/shuffle/RoundRobinPartitioner.h +++ b/cpp/core/shuffle/RoundRobinPartitioner.h @@ -32,6 +32,12 @@ class RoundRobinPartitioner final : public Partitioner { std::vector& row2Partition, std::vector& partition2RowCount) override; + arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) override; + private: friend class RoundRobinPartitionerTest; diff --git a/cpp/core/shuffle/ShuffleReader.cc b/cpp/core/shuffle/ShuffleReader.cc index 471409d6d4b2..faa81b52206a 100644 --- a/cpp/core/shuffle/ShuffleReader.cc +++ b/cpp/core/shuffle/ShuffleReader.cc @@ -48,6 +48,10 @@ int64_t ShuffleReader::getIpcTime() const { return ipcTime_; } +ShuffleWriterType ShuffleReader::getShuffleWriterType() const { + return factory_->getShuffleWriterType(); +} + int64_t ShuffleReader::getDeserializeTime() const { return factory_->getDeserializeTime(); } diff --git a/cpp/core/shuffle/ShuffleReader.h b/cpp/core/shuffle/ShuffleReader.h index 4ba1057129ce..5cef14768450 100644 --- a/cpp/core/shuffle/ShuffleReader.h +++ b/cpp/core/shuffle/ShuffleReader.h @@ -39,6 +39,8 @@ class DeserializerFactory { virtual int64_t getDecompressTime() = 0; virtual int64_t getDeserializeTime() = 0; + + virtual ShuffleWriterType getShuffleWriterType() = 0; }; class ShuffleReader { @@ -60,13 +62,15 @@ class ShuffleReader { arrow::MemoryPool* getPool() const; + ShuffleWriterType getShuffleWriterType() const; + protected: arrow::MemoryPool* pool_; int64_t decompressTime_ = 0; int64_t ipcTime_ = 0; int64_t deserializeTime_ = 0; - ShuffleReaderOptions options_; + ShuffleWriterType shuffleWriterType_; private: std::shared_ptr schema_; diff --git a/cpp/core/shuffle/ShuffleWriter.h b/cpp/core/shuffle/ShuffleWriter.h index bcf0c2c3b78d..a7987ce3e4da 100644 --- a/cpp/core/shuffle/ShuffleWriter.h +++ b/cpp/core/shuffle/ShuffleWriter.h @@ -37,7 +37,7 @@ class ShuffleWriter : public Reclaimable { public: static constexpr int64_t kMinMemLimit = 128LL * 1024 * 1024; - virtual arrow::Status split(std::shared_ptr cb, int64_t memLimit) = 0; + virtual arrow::Status write(std::shared_ptr cb, int64_t memLimit) = 0; virtual arrow::Status stop() = 0; @@ -45,6 +45,10 @@ class ShuffleWriter : public Reclaimable { return numPartitions_; } + ShuffleWriterOptions& options() { + return options_; + } + int64_t partitionBufferSize() const { return partitionBufferPool_->bytes_allocated(); } @@ -81,7 +85,9 @@ class ShuffleWriter : public Reclaimable { return metrics_.rawPartitionLengths; } - virtual const uint64_t cachedPayloadSize() const = 0; + const int64_t rawPartitionBytes() { + return std::accumulate(metrics_.rawPartitionLengths.begin(), metrics_.rawPartitionLengths.end(), 0LL); + } protected: ShuffleWriter( @@ -108,6 +114,8 @@ class ShuffleWriter : public Reclaimable { std::unique_ptr partitionWriter_; + std::vector rowVectorLengths_; + std::shared_ptr schema_; // Column index, partition id, buffers. diff --git a/cpp/core/shuffle/SinglePartitioner.cc b/cpp/core/shuffle/SinglePartitioner.cc index c4f80ce798b2..981a5b8e453c 100644 --- a/cpp/core/shuffle/SinglePartitioner.cc +++ b/cpp/core/shuffle/SinglePartitioner.cc @@ -28,4 +28,13 @@ arrow::Status gluten::SinglePartitioner::compute( return arrow::Status::OK(); } +arrow::Status gluten::SinglePartitioner::compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) { + // nothing is need do here + return arrow::Status::OK(); +} + } // namespace gluten diff --git a/cpp/core/shuffle/SinglePartitioner.h b/cpp/core/shuffle/SinglePartitioner.h index d3d2c29f76bf..e5d7a920fd6c 100644 --- a/cpp/core/shuffle/SinglePartitioner.h +++ b/cpp/core/shuffle/SinglePartitioner.h @@ -29,5 +29,11 @@ class SinglePartitioner final : public Partitioner { const int64_t numRows, std::vector& row2partition, std::vector& partition2RowCount) override; + + arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) override; }; } // namespace gluten diff --git a/cpp/core/shuffle/rss/RssClient.h b/cpp/core/shuffle/rss/RssClient.h index 9209430b07d3..dddccfa1ad52 100644 --- a/cpp/core/shuffle/rss/RssClient.h +++ b/cpp/core/shuffle/rss/RssClient.h @@ -21,7 +21,7 @@ class RssClient { public: virtual ~RssClient() = default; - virtual int32_t pushPartitionData(int32_t partitionId, char* bytes, int64_t size) = 0; + virtual int32_t pushPartitionData(int32_t partitionId, const char* bytes, int64_t size) = 0; virtual void stop() = 0; }; diff --git a/cpp/core/shuffle/rss/RssPartitionWriter.cc b/cpp/core/shuffle/rss/RssPartitionWriter.cc index 15981bf8ded8..015129e26ed2 100644 --- a/cpp/core/shuffle/rss/RssPartitionWriter.cc +++ b/cpp/core/shuffle/rss/RssPartitionWriter.cc @@ -73,4 +73,11 @@ arrow::Status RssPartitionWriter::evict( partitionId, reinterpret_cast(const_cast(buffer->data())), buffer->size()); return arrow::Status::OK(); } + +arrow::Status RssPartitionWriter::evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) { + rawPartitionLengths_[partitionId] += rawSize; + ScopedTimer timer(&spillTime_); + bytesEvicted_[partitionId] += rssClient_->pushPartitionData(partitionId, data, length); + return arrow::Status::OK(); +} } // namespace gluten diff --git a/cpp/core/shuffle/rss/RssPartitionWriter.h b/cpp/core/shuffle/rss/RssPartitionWriter.h index ef43017fcf64..b8cc1551c0cd 100644 --- a/cpp/core/shuffle/rss/RssPartitionWriter.h +++ b/cpp/core/shuffle/rss/RssPartitionWriter.h @@ -44,6 +44,8 @@ class RssPartitionWriter final : public RemotePartitionWriter { bool reuseBuffers, bool hasComplexType) override; + arrow::Status evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) override; + arrow::Status reclaimFixedSize(int64_t size, int64_t* actual) override; arrow::Status stop(ShuffleWriterMetrics* metrics) override; diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 9e5f08b1cb46..c058883b603f 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -310,7 +310,8 @@ set(VELOX_SRCS operators/serializer/VeloxRowToColumnarConverter.cc operators/writer/VeloxParquetDatasource.cc shuffle/VeloxShuffleReader.cc - shuffle/VeloxShuffleWriter.cc + shuffle/VeloxHashBasedShuffleWriter.cc + shuffle/VeloxSortBasedShuffleWriter.cc substrait/SubstraitParser.cc substrait/SubstraitToVeloxExpr.cc substrait/SubstraitToVeloxPlan.cc diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc index 14593c8dfa12..71d3d96b5330 100644 --- a/cpp/velox/benchmarks/GenericBenchmark.cc +++ b/cpp/velox/benchmarks/GenericBenchmark.cc @@ -31,6 +31,7 @@ #include "compute/VeloxRuntime.h" #include "config/GlutenConfig.h" #include "shuffle/LocalPartitionWriter.h" +#include "shuffle/VeloxHashBasedShuffleWriter.h" #include "shuffle/VeloxShuffleWriter.h" #include "shuffle/rss/RssPartitionWriter.h" #include "utils/StringUtil.h" @@ -111,7 +112,7 @@ std::shared_ptr createShuffleWriter( options.partitioning = gluten::toPartitioning(FLAGS_partitioning); GLUTEN_ASSIGN_OR_THROW( auto shuffleWriter, - VeloxShuffleWriter::create( + VeloxHashBasedShuffleWriter::create( FLAGS_shuffle_partitions, std::move(partitionWriter), std::move(options), @@ -191,7 +192,7 @@ auto BM_Generic = [](::benchmark::State& state, GLUTEN_THROW_NOT_OK(setLocalDirsAndDataFileFromEnv(dataFile, localDirs, isFromEnv)); const auto& shuffleWriter = createShuffleWriter(memoryManager.get(), dataFile, localDirs); while (resultIter->hasNext()) { - GLUTEN_THROW_NOT_OK(shuffleWriter->split(resultIter->next(), ShuffleWriter::kMinMemLimit)); + GLUTEN_THROW_NOT_OK(shuffleWriter->write(resultIter->next(), ShuffleWriter::kMinMemLimit)); } GLUTEN_THROW_NOT_OK(shuffleWriter->stop()); TIME_NANO_END(shuffleWriteTime); diff --git a/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc b/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc index 0109de603b2a..4a4bb69b8d78 100644 --- a/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc +++ b/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc @@ -31,6 +31,7 @@ #include "benchmarks/common/BenchmarkUtils.h" #include "memory/ColumnarBatch.h" #include "shuffle/LocalPartitionWriter.h" +#include "shuffle/VeloxHashBasedShuffleWriter.h" #include "shuffle/VeloxShuffleWriter.h" #include "utils/TestUtils.h" #include "utils/VeloxArrowUtils.h" @@ -259,7 +260,7 @@ class BenchmarkShuffleSplitCacheScanBenchmark : public BenchmarkShuffleSplit { numPartitions, PartitionWriterOptions{}, defaultArrowMemoryPool().get(), dataFile, localDirs); GLUTEN_ASSIGN_OR_THROW( shuffleWriter, - VeloxShuffleWriter::create( + VeloxHashBasedShuffleWriter::create( numPartitions, std::move(partitionWriter), std::move(options), @@ -294,7 +295,7 @@ class BenchmarkShuffleSplitCacheScanBenchmark : public BenchmarkShuffleSplit { [&shuffleWriter, &splitTime](const std::shared_ptr& recordBatch) { std::shared_ptr cb; ARROW_ASSIGN_OR_THROW(cb, recordBatch2VeloxColumnarBatch(*recordBatch)); - TIME_NANO_OR_THROW(splitTime, shuffleWriter->split(cb, ShuffleWriter::kMinMemLimit)); + TIME_NANO_OR_THROW(splitTime, shuffleWriter->write(cb, ShuffleWriter::kMinMemLimit)); }); // LOG(INFO) << " split done memory allocated = " << // options.memoryPool->bytes_allocated(); @@ -327,7 +328,7 @@ class BenchmarkShuffleSplitIterateScanBenchmark : public BenchmarkShuffleSplit { numPartitions, PartitionWriterOptions{}, defaultArrowMemoryPool().get(), dataFile, localDirs); GLUTEN_ASSIGN_OR_THROW( shuffleWriter, - VeloxShuffleWriter::create( + VeloxHashBasedShuffleWriter::create( numPartitions, std::move(partitionWriter), std::move(options), @@ -350,7 +351,7 @@ class BenchmarkShuffleSplitIterateScanBenchmark : public BenchmarkShuffleSplit { numRows += recordBatch->num_rows(); std::shared_ptr cb; ARROW_ASSIGN_OR_THROW(cb, recordBatch2VeloxColumnarBatch(*recordBatch)); - TIME_NANO_OR_THROW(splitTime, shuffleWriter->split(cb, ShuffleWriter::kMinMemLimit)); + TIME_NANO_OR_THROW(splitTime, shuffleWriter->write(cb, ShuffleWriter::kMinMemLimit)); TIME_NANO_OR_THROW(elapseRead, recordBatchReader->ReadNext(&recordBatch)); } } diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index a3e8c159c9ee..15c84b41cdad 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -28,8 +28,9 @@ #include "compute/VeloxPlanConverter.h" #include "config/VeloxConfig.h" #include "operators/serializer/VeloxRowToColumnarConverter.h" +#include "shuffle/VeloxHashBasedShuffleWriter.h" #include "shuffle/VeloxShuffleReader.h" -#include "shuffle/VeloxShuffleWriter.h" +#include "shuffle/VeloxSortBasedShuffleWriter.h" #include "utils/ConfigExtractor.h" #include "utils/VeloxArrowUtils.h" @@ -187,10 +188,19 @@ std::shared_ptr VeloxRuntime::createShuffleWriter( MemoryManager* memoryManager) { auto ctxPool = getLeafVeloxPool(memoryManager); auto arrowPool = memoryManager->getArrowMemoryPool(); - GLUTEN_ASSIGN_OR_THROW( - auto shuffle_writer, - VeloxShuffleWriter::create(numPartitions, std::move(partitionWriter), std::move(options), ctxPool, arrowPool)); - return shuffle_writer; + std::shared_ptr shuffleWriter; + if (options.shuffleWriterType == kHashShuffle) { + GLUTEN_ASSIGN_OR_THROW( + shuffleWriter, + VeloxHashBasedShuffleWriter::create( + numPartitions, std::move(partitionWriter), std::move(options), ctxPool, arrowPool)); + } else if (options.shuffleWriterType == kSortShuffle) { + GLUTEN_ASSIGN_OR_THROW( + shuffleWriter, + VeloxSortBasedShuffleWriter::create( + numPartitions, std::move(partitionWriter), std::move(options), ctxPool, arrowPool)); + } + return shuffleWriter; } std::shared_ptr VeloxRuntime::createDatasource( @@ -242,9 +252,18 @@ std::shared_ptr VeloxRuntime::createShuffleReader( auto rowType = facebook::velox::asRowType(gluten::fromArrowSchema(schema)); auto codec = gluten::createArrowIpcCodec(options.compressionType, options.codecBackend); auto ctxVeloxPool = getLeafVeloxPool(memoryManager); + auto veloxCompressionType = facebook::velox::common::stringToCompressionKind(options.compressionTypeStr); auto deserializerFactory = std::make_unique( - schema, std::move(codec), rowType, options.batchSize, pool, ctxVeloxPool); - return std::make_shared(std::move(deserializerFactory)); + schema, + std::move(codec), + veloxCompressionType, + rowType, + options.batchSize, + pool, + ctxVeloxPool, + options.shuffleWriterType); + auto reader = std::make_shared(std::move(deserializerFactory)); + return reader; } std::unique_ptr VeloxRuntime::createColumnarBatchSerializer( diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.cc b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc similarity index 90% rename from cpp/velox/shuffle/VeloxShuffleWriter.cc rename to cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc index b304565e5d05..daff1370332f 100644 --- a/cpp/velox/shuffle/VeloxShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "VeloxShuffleWriter.h" +#include "VeloxHashBasedShuffleWriter.h" #include "memory/ArrowMemory.h" #include "memory/VeloxColumnarBatch.h" #include "memory/VeloxMemoryManager.h" @@ -70,58 +70,6 @@ bool vectorHasNull(const facebook::velox::VectorPtr& vp) { return vp->countNulls(vp->nulls(), vp->size()) != 0; } -facebook::velox::RowVectorPtr getStrippedRowVector(const facebook::velox::RowVector& rv) { - // get new row type - auto rowType = rv.type()->asRow(); - auto typeChildren = rowType.children(); - typeChildren.erase(typeChildren.begin()); - auto newRowType = facebook::velox::ROW(std::move(typeChildren)); - - // get length - auto length = rv.size(); - - // get children - auto children = rv.children(); - children.erase(children.begin()); - - return std::make_shared( - rv.pool(), newRowType, facebook::velox::BufferPtr(nullptr), length, std::move(children)); -} - -const int32_t* getFirstColumn(const facebook::velox::RowVector& rv) { - VELOX_CHECK(rv.childrenSize() > 0, "RowVector missing partition id column."); - - auto& firstChild = rv.childAt(0); - VELOX_CHECK(firstChild->isFlatEncoding(), "Partition id (field 0) is not flat encoding."); - VELOX_CHECK( - firstChild->type()->isInteger(), - "Partition id (field 0) should be integer, but got {}", - firstChild->type()->toString()); - - // first column is partition key hash value or pid - return firstChild->asFlatVector()->rawValues(); -} - -class EvictGuard { - public: - explicit EvictGuard(EvictState& evictState) : evictState_(evictState) { - evictState_ = EvictState::kUnevictable; - } - - ~EvictGuard() { - evictState_ = EvictState::kEvictable; - } - - // For safety and clarity. - EvictGuard(const EvictGuard&) = delete; - EvictGuard& operator=(const EvictGuard&) = delete; - EvictGuard(EvictGuard&&) = delete; - EvictGuard& operator=(EvictGuard&&) = delete; - - private: - EvictState& evictState_; -}; - class BinaryArrayResizeGuard { public: explicit BinaryArrayResizeGuard(BinaryArrayResizeState& state) : state_(state) { @@ -199,19 +147,19 @@ arrow::Status collectFlatVectorBuffer( } // namespace -arrow::Result> VeloxShuffleWriter::create( +arrow::Result> VeloxHashBasedShuffleWriter::create( uint32_t numPartitions, std::unique_ptr partitionWriter, ShuffleWriterOptions options, std::shared_ptr veloxPool, arrow::MemoryPool* arrowPool) { - std::shared_ptr res( - new VeloxShuffleWriter(numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool)); + std::shared_ptr res(new VeloxHashBasedShuffleWriter( + numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool)); RETURN_NOT_OK(res->init()); return res; } // namespace gluten -arrow::Status VeloxShuffleWriter::init() { +arrow::Status VeloxHashBasedShuffleWriter::init() { #if defined(__x86_64__) supportAvx512_ = __builtin_cpu_supports("avx512bw"); #else @@ -235,7 +183,7 @@ arrow::Status VeloxShuffleWriter::init() { return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::initPartitions() { +arrow::Status VeloxHashBasedShuffleWriter::initPartitions() { auto simpleColumnCount = simpleColumnIndices_.size(); partitionValidityAddrs_.resize(simpleColumnCount); @@ -260,15 +208,11 @@ arrow::Status VeloxShuffleWriter::initPartitions() { return arrow::Status::OK(); } -int64_t VeloxShuffleWriter::rawPartitionBytes() const { - return std::accumulate(metrics_.rawPartitionLengths.begin(), metrics_.rawPartitionLengths.end(), 0LL); -} - -void VeloxShuffleWriter::setPartitionBufferSize(uint32_t newSize) { +void VeloxHashBasedShuffleWriter::setPartitionBufferSize(uint32_t newSize) { options_.bufferSize = newSize; } -arrow::Result> VeloxShuffleWriter::generateComplexTypeBuffers( +arrow::Result> VeloxHashBasedShuffleWriter::generateComplexTypeBuffers( facebook::velox::RowVectorPtr vector) { auto arena = std::make_unique(veloxPool_.get()); auto serializer = @@ -291,7 +235,7 @@ arrow::Result> VeloxShuffleWriter::generateComple return valueBuffer; } -arrow::Status VeloxShuffleWriter::split(std::shared_ptr cb, int64_t memLimit) { +arrow::Status VeloxHashBasedShuffleWriter::write(std::shared_ptr cb, int64_t memLimit) { if (options_.partitioning == Partitioning::kSingle) { auto veloxColumnBatch = VeloxColumnarBatch::from(veloxPool_.get(), cb); VELOX_CHECK_NOT_NULL(veloxColumnBatch); @@ -357,7 +301,7 @@ arrow::Status VeloxShuffleWriter::split(std::shared_ptr cb, int64 return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::partitioningAndDoSplit(facebook::velox::RowVectorPtr rv, int64_t memLimit) { +arrow::Status VeloxHashBasedShuffleWriter::partitioningAndDoSplit(facebook::velox::RowVectorPtr rv, int64_t memLimit) { if (partitioner_->hasPid()) { auto pidArr = getFirstColumn(*rv); START_TIMING(cpuWallTimingList_[CpuWallTimingCompute]); @@ -376,7 +320,7 @@ arrow::Status VeloxShuffleWriter::partitioningAndDoSplit(facebook::velox::RowVec return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::stop() { +arrow::Status VeloxHashBasedShuffleWriter::stop() { if (options_.partitioning != Partitioning::kSingle) { for (auto pid = 0; pid < numPartitions_; ++pid) { RETURN_NOT_OK(evictPartitionBuffers(pid, false)); @@ -394,7 +338,7 @@ arrow::Status VeloxShuffleWriter::stop() { return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::buildPartition2Row(uint32_t rowNum) { +arrow::Status VeloxHashBasedShuffleWriter::buildPartition2Row(uint32_t rowNum) { SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingBuildPartition]); // calc partition2RowOffsetBase_ @@ -427,7 +371,7 @@ arrow::Status VeloxShuffleWriter::buildPartition2Row(uint32_t rowNum) { return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::updateInputHasNull(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::updateInputHasNull(const facebook::velox::RowVector& rv) { SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingHasNull]); for (size_t col = 0; col < simpleColumnIndices_.size(); ++col) { @@ -444,11 +388,11 @@ arrow::Status VeloxShuffleWriter::updateInputHasNull(const facebook::velox::RowV return arrow::Status::OK(); } -void VeloxShuffleWriter::setSplitState(SplitState state) { +void VeloxHashBasedShuffleWriter::setSplitState(SplitState state) { splitState_ = state; } -arrow::Status VeloxShuffleWriter::doSplit(const facebook::velox::RowVector& rv, int64_t memLimit) { +arrow::Status VeloxHashBasedShuffleWriter::doSplit(const facebook::velox::RowVector& rv, int64_t memLimit) { auto rowNum = rv.size(); RETURN_NOT_OK(buildPartition2Row(rowNum)); RETURN_NOT_OK(updateInputHasNull(rv)); @@ -472,7 +416,7 @@ arrow::Status VeloxShuffleWriter::doSplit(const facebook::velox::RowVector& rv, return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitRowVector(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::splitRowVector(const facebook::velox::RowVector& rv) { SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingSplitRV]); // now start to split the RowVector @@ -489,7 +433,7 @@ arrow::Status VeloxShuffleWriter::splitRowVector(const facebook::velox::RowVecto return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitFixedWidthValueBuffer(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::splitFixedWidthValueBuffer(const facebook::velox::RowVector& rv) { for (auto col = 0; col < fixedWidthColumnCount_; ++col) { auto colIdx = simpleColumnIndices_[col]; auto& column = rv.childAt(colIdx); @@ -543,7 +487,9 @@ arrow::Status VeloxShuffleWriter::splitFixedWidthValueBuffer(const facebook::vel return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitBoolType(const uint8_t* srcAddr, const std::vector& dstAddrs) { +arrow::Status VeloxHashBasedShuffleWriter::splitBoolType( + const uint8_t* srcAddr, + const std::vector& dstAddrs) { // assume batch size = 32k; reducer# = 4K; row/reducer = 8 for (auto& pid : partitionUsed_) { // set the last byte @@ -632,7 +578,7 @@ arrow::Status VeloxShuffleWriter::splitBoolType(const uint8_t* srcAddr, const st return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitValidityBuffer(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::splitValidityBuffer(const facebook::velox::RowVector& rv) { for (size_t col = 0; col < simpleColumnIndices_.size(); ++col) { auto colIdx = simpleColumnIndices_[col]; auto& column = rv.childAt(colIdx); @@ -660,7 +606,7 @@ arrow::Status VeloxShuffleWriter::splitValidityBuffer(const facebook::velox::Row return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitBinaryType( +arrow::Status VeloxHashBasedShuffleWriter::splitBinaryType( uint32_t binaryIdx, const facebook::velox::FlatVector& src, std::vector& dst) { @@ -723,7 +669,7 @@ arrow::Status VeloxShuffleWriter::splitBinaryType( return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitBinaryArray(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::splitBinaryArray(const facebook::velox::RowVector& rv) { for (auto col = fixedWidthColumnCount_; col < simpleColumnIndices_.size(); ++col) { auto binaryIdx = col - fixedWidthColumnCount_; auto& dstAddrs = partitionBinaryAddrs_[binaryIdx]; @@ -734,7 +680,7 @@ arrow::Status VeloxShuffleWriter::splitBinaryArray(const facebook::velox::RowVec return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::splitComplexType(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::splitComplexType(const facebook::velox::RowVector& rv) { if (complexColumnIndices_.size() == 0) { return arrow::Status::OK(); } @@ -773,7 +719,7 @@ arrow::Status VeloxShuffleWriter::splitComplexType(const facebook::velox::RowVec return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::initColumnTypes(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::initColumnTypes(const facebook::velox::RowVector& rv) { schema_ = toArrowSchema(rv.type(), veloxPool_.get()); for (size_t i = 0; i < rv.childrenSize(); ++i) { veloxColumnTypes_.push_back(rv.childAt(i)->type()); @@ -837,7 +783,7 @@ arrow::Status VeloxShuffleWriter::initColumnTypes(const facebook::velox::RowVect return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::initFromRowVector(const facebook::velox::RowVector& rv) { +arrow::Status VeloxHashBasedShuffleWriter::initFromRowVector(const facebook::velox::RowVector& rv) { if (veloxColumnTypes_.empty()) { RETURN_NOT_OK(initColumnTypes(rv)); RETURN_NOT_OK(initPartitions()); @@ -846,13 +792,13 @@ arrow::Status VeloxShuffleWriter::initFromRowVector(const facebook::velox::RowVe return arrow::Status::OK(); } -inline bool VeloxShuffleWriter::beyondThreshold(uint32_t partitionId, uint32_t newSize) { +inline bool VeloxHashBasedShuffleWriter::beyondThreshold(uint32_t partitionId, uint32_t newSize) { auto currentBufferSize = partitionBufferSize_[partitionId]; return newSize > (1 + options_.bufferReallocThreshold) * currentBufferSize || newSize < (1 - options_.bufferReallocThreshold) * currentBufferSize; } -void VeloxShuffleWriter::calculateSimpleColumnBytes() { +void VeloxHashBasedShuffleWriter::calculateSimpleColumnBytes() { fixedWidthBufferBytes_ = 0; for (size_t col = 0; col < fixedWidthColumnCount_; ++col) { auto colIdx = simpleColumnIndices_[col]; @@ -862,7 +808,9 @@ void VeloxShuffleWriter::calculateSimpleColumnBytes() { fixedWidthBufferBytes_ += kSizeOfBinaryArrayLengthBuffer * binaryColumnIndices_.size(); } -uint32_t VeloxShuffleWriter::calculatePartitionBufferSize(const facebook::velox::RowVector& rv, int64_t memLimit) { +uint32_t VeloxHashBasedShuffleWriter::calculatePartitionBufferSize( + const facebook::velox::RowVector& rv, + int64_t memLimit) { auto bytesPerRow = fixedWidthBufferBytes_; SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingCalculateBufferSize]); @@ -915,7 +863,7 @@ uint32_t VeloxShuffleWriter::calculatePartitionBufferSize(const facebook::velox: } arrow::Result> -VeloxShuffleWriter::allocateValidityBuffer(uint32_t col, uint32_t partitionId, uint32_t newSize) { +VeloxHashBasedShuffleWriter::allocateValidityBuffer(uint32_t col, uint32_t partitionId, uint32_t newSize) { if (inputHasNull_[col]) { ARROW_ASSIGN_OR_RAISE( auto validityBuffer, @@ -929,7 +877,7 @@ VeloxShuffleWriter::allocateValidityBuffer(uint32_t col, uint32_t partitionId, u return nullptr; } -arrow::Status VeloxShuffleWriter::updateValidityBuffers(uint32_t partitionId, uint32_t newSize) { +arrow::Status VeloxHashBasedShuffleWriter::updateValidityBuffers(uint32_t partitionId, uint32_t newSize) { for (auto i = 0; i < simpleColumnIndices_.size(); ++i) { // If the validity buffer is not yet allocated, allocate and fill 0xff based on inputHasNull_. if (partitionValidityAddrs_[i][partitionId] == nullptr) { @@ -940,7 +888,7 @@ arrow::Status VeloxShuffleWriter::updateValidityBuffers(uint32_t partitionId, ui return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::allocatePartitionBuffer(uint32_t partitionId, uint32_t newSize) { +arrow::Status VeloxHashBasedShuffleWriter::allocatePartitionBuffer(uint32_t partitionId, uint32_t newSize) { SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingAllocateBuffer]); for (auto i = 0; i < simpleColumnIndices_.size(); ++i) { @@ -987,7 +935,7 @@ arrow::Status VeloxShuffleWriter::allocatePartitionBuffer(uint32_t partitionId, return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::evictBuffers( +arrow::Status VeloxHashBasedShuffleWriter::evictBuffers( uint32_t partitionId, uint32_t numRows, std::vector> buffers, @@ -1000,7 +948,7 @@ arrow::Status VeloxShuffleWriter::evictBuffers( return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::evictPartitionBuffers(uint32_t partitionId, bool reuseBuffers) { +arrow::Status VeloxHashBasedShuffleWriter::evictPartitionBuffers(uint32_t partitionId, bool reuseBuffers) { auto numRows = partitionBufferBase_[partitionId]; if (numRows > 0) { ARROW_ASSIGN_OR_RAISE(auto buffers, assembleBuffers(partitionId, reuseBuffers)); @@ -1009,7 +957,7 @@ arrow::Status VeloxShuffleWriter::evictPartitionBuffers(uint32_t partitionId, bo return arrow::Status::OK(); } -arrow::Result>> VeloxShuffleWriter::assembleBuffers( +arrow::Result>> VeloxHashBasedShuffleWriter::assembleBuffers( uint32_t partitionId, bool reuseBuffers) { SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingCreateRbFromBuffer]); @@ -1150,7 +1098,7 @@ arrow::Result>> VeloxShuffleWriter::a return allBuffers; } -arrow::Status VeloxShuffleWriter::reclaimFixedSize(int64_t size, int64_t* actual) { +arrow::Status VeloxHashBasedShuffleWriter::reclaimFixedSize(int64_t size, int64_t* actual) { if (evictState_ == EvictState::kUnevictable) { *actual = 0; return arrow::Status::OK(); @@ -1174,7 +1122,7 @@ arrow::Status VeloxShuffleWriter::reclaimFixedSize(int64_t size, int64_t* actual return arrow::Status::OK(); } -arrow::Result VeloxShuffleWriter::evictCachedPayload(int64_t size) { +arrow::Result VeloxHashBasedShuffleWriter::evictCachedPayload(int64_t size) { SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingEvictPartition]); int64_t actual; auto before = partitionBufferPool_->bytes_allocated(); @@ -1188,7 +1136,7 @@ arrow::Result VeloxShuffleWriter::evictCachedPayload(int64_t size) { return actual; } -arrow::Status VeloxShuffleWriter::resetValidityBuffer(uint32_t partitionId) { +arrow::Status VeloxHashBasedShuffleWriter::resetValidityBuffer(uint32_t partitionId) { std::for_each(partitionBuffers_.begin(), partitionBuffers_.end(), [partitionId](auto& bufs) { if (bufs[partitionId].size() != 0 && bufs[partitionId][kValidityBufferIndex] != nullptr) { // initialize all true once allocated @@ -1199,7 +1147,8 @@ arrow::Status VeloxShuffleWriter::resetValidityBuffer(uint32_t partitionId) { return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::resizePartitionBuffer(uint32_t partitionId, uint32_t newSize, bool preserveData) { +arrow::Status +VeloxHashBasedShuffleWriter::resizePartitionBuffer(uint32_t partitionId, uint32_t newSize, bool preserveData) { for (auto i = 0; i < simpleColumnIndices_.size(); ++i) { auto columnType = schema_->field(simpleColumnIndices_[i])->type()->id(); auto& buffers = partitionBuffers_[i][partitionId]; @@ -1278,7 +1227,7 @@ arrow::Status VeloxShuffleWriter::resizePartitionBuffer(uint32_t partitionId, ui return arrow::Status::OK(); } -arrow::Status VeloxShuffleWriter::shrinkPartitionBuffer(uint32_t partitionId) { +arrow::Status VeloxHashBasedShuffleWriter::shrinkPartitionBuffer(uint32_t partitionId) { auto bufferSize = partitionBufferSize_[partitionId]; if (bufferSize == 0) { return arrow::Status::OK(); @@ -1301,11 +1250,11 @@ arrow::Status VeloxShuffleWriter::shrinkPartitionBuffer(uint32_t partitionId) { return resizePartitionBuffer(partitionId, newSize, /*preserveData=*/true); } -uint64_t VeloxShuffleWriter::valueBufferSizeForBinaryArray(uint32_t binaryIdx, uint32_t newSize) { +uint64_t VeloxHashBasedShuffleWriter::valueBufferSizeForBinaryArray(uint32_t binaryIdx, uint32_t newSize) { return (binaryArrayTotalSizeBytes_[binaryIdx] + totalInputNumRows_ - 1) / totalInputNumRows_ * newSize + 1024; } -uint64_t VeloxShuffleWriter::valueBufferSizeForFixedWidthArray(uint32_t fixedWidthIndex, uint32_t newSize) { +uint64_t VeloxHashBasedShuffleWriter::valueBufferSizeForFixedWidthArray(uint32_t fixedWidthIndex, uint32_t newSize) { uint64_t valueBufferSize = 0; auto columnIdx = simpleColumnIndices_[fixedWidthIndex]; if (arrowColumnTypes_[columnIdx]->id() == arrow::BooleanType::type_id) { @@ -1320,7 +1269,7 @@ uint64_t VeloxShuffleWriter::valueBufferSizeForFixedWidthArray(uint32_t fixedWid return valueBufferSize; } -void VeloxShuffleWriter::stat() const { +void VeloxHashBasedShuffleWriter::stat() const { #if VELOX_SHUFFLE_WRITER_LOG_FLAG for (int i = CpuWallTimingBegin; i != CpuWallTimingEnd; ++i) { std::ostringstream oss; @@ -1336,7 +1285,7 @@ void VeloxShuffleWriter::stat() const { #endif } -arrow::Status VeloxShuffleWriter::resetPartitionBuffer(uint32_t partitionId) { +arrow::Status VeloxHashBasedShuffleWriter::resetPartitionBuffer(uint32_t partitionId) { // Reset fixed-width partition buffers for (auto i = 0; i < fixedWidthColumnCount_; ++i) { partitionValidityAddrs_[i][partitionId] = nullptr; @@ -1356,11 +1305,11 @@ arrow::Status VeloxShuffleWriter::resetPartitionBuffer(uint32_t partitionId) { return arrow::Status::OK(); } -const uint64_t VeloxShuffleWriter::cachedPayloadSize() const { +const uint64_t VeloxHashBasedShuffleWriter::cachedPayloadSize() const { return partitionWriter_->cachedPayloadSize(); } -arrow::Result VeloxShuffleWriter::shrinkPartitionBuffersMinSize(int64_t size) { +arrow::Result VeloxHashBasedShuffleWriter::shrinkPartitionBuffersMinSize(int64_t size) { // Sort partition buffers by (partitionBufferSize_ - partitionBufferBase_) std::vector> pidToSize; for (auto pid = 0; pid < numPartitions_; ++pid) { @@ -1388,7 +1337,7 @@ arrow::Result VeloxShuffleWriter::shrinkPartitionBuffersMinSize(int64_t return shrunken; } -arrow::Result VeloxShuffleWriter::evictPartitionBuffersMinSize(int64_t size) { +arrow::Result VeloxHashBasedShuffleWriter::evictPartitionBuffersMinSize(int64_t size) { // Evict partition buffers, only when splitState_ == SplitState::kInit, and space freed from // shrinking is not enough. In this case partitionBufferSize_ == partitionBufferBase_ int64_t beforeEvict = partitionBufferPool_->bytes_allocated(); @@ -1415,7 +1364,7 @@ arrow::Result VeloxShuffleWriter::evictPartitionBuffersMinSize(int64_t return evicted; } -bool VeloxShuffleWriter::shrinkPartitionBuffersAfterSpill() const { +bool VeloxHashBasedShuffleWriter::shrinkPartitionBuffersAfterSpill() const { // If OOM happens during SplitState::kSplit, it is triggered by binary buffers resize. // Or during SplitState::kInit, it is triggered by other operators. // The reclaim order is spill->shrink, because the partition buffers can be reused. @@ -1424,13 +1373,13 @@ bool VeloxShuffleWriter::shrinkPartitionBuffersAfterSpill() const { (splitState_ == SplitState::kSplit || splitState_ == SplitState::kInit); } -bool VeloxShuffleWriter::evictPartitionBuffersAfterSpill() const { +bool VeloxHashBasedShuffleWriter::evictPartitionBuffersAfterSpill() const { // If OOM triggered by other operators, the splitState_ is SplitState::kInit. // The last resort is to evict the partition buffers to reclaim more space. return options_.partitioning != Partitioning::kSingle && splitState_ == SplitState::kInit; } -arrow::Result VeloxShuffleWriter::partitionBufferSizeAfterShrink(uint32_t partitionId) const { +arrow::Result VeloxHashBasedShuffleWriter::partitionBufferSizeAfterShrink(uint32_t partitionId) const { if (splitState_ == SplitState::kSplit) { return partitionBufferBase_[partitionId] + partition2RowCount_[partitionId]; } @@ -1440,7 +1389,7 @@ arrow::Result VeloxShuffleWriter::partitionBufferSizeAfterShrink(uint3 return arrow::Status::Invalid("Cannot shrink partition buffers in SplitState: " + std::to_string(splitState_)); } -arrow::Status VeloxShuffleWriter::preAllocPartitionBuffers(uint32_t preAllocBufferSize) { +arrow::Status VeloxHashBasedShuffleWriter::preAllocPartitionBuffers(uint32_t preAllocBufferSize) { for (auto& pid : partitionUsed_) { auto newSize = std::max(preAllocBufferSize, partition2RowCount_[pid]); VLOG_IF(9, partitionBufferSize_[pid] != newSize) @@ -1494,7 +1443,7 @@ arrow::Status VeloxShuffleWriter::preAllocPartitionBuffers(uint32_t preAllocBuff return arrow::Status::OK(); } -bool VeloxShuffleWriter::isExtremelyLargeBatch(facebook::velox::RowVectorPtr& rv) const { +bool VeloxHashBasedShuffleWriter::isExtremelyLargeBatch(facebook::velox::RowVectorPtr& rv) const { return (rv->size() > maxBatchSize_ && maxBatchSize_ > 0); } diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h new file mode 100644 index 000000000000..a11f84e952a6 --- /dev/null +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "velox/common/time/CpuWallTimer.h" +#include "velox/serializers/PrestoSerializer.h" +#include "velox/type/Type.h" +#include "velox/vector/ComplexVector.h" +#include "velox/vector/FlatVector.h" +#include "velox/vector/VectorStream.h" + +#include +#include +#include +#include +#include +#include + +#include "VeloxShuffleWriter.h" +#include "memory/VeloxMemoryManager.h" +#include "shuffle/PartitionWriter.h" +#include "shuffle/Partitioner.h" +#include "shuffle/Utils.h" + +#include "utils/Print.h" + +namespace gluten { + +// set 1 to open print +#define VELOX_SHUFFLE_WRITER_PRINT 0 + +#if VELOX_SHUFFLE_WRITER_PRINT + +#define VsPrint Print +#define VsPrintLF PrintLF +#define VsPrintSplit PrintSplit +#define VsPrintSplitLF PrintSplitLF +#define VsPrintVectorRange PrintVectorRange +#define VS_PRINT PRINT +#define VS_PRINTLF PRINTLF +#define VS_PRINT_FUNCTION_NAME PRINT_FUNCTION_NAME +#define VS_PRINT_FUNCTION_SPLIT_LINE PRINT_FUNCTION_SPLIT_LINE +#define VS_PRINT_CONTAINER PRINT_CONTAINER +#define VS_PRINT_CONTAINER_TO_STRING PRINT_CONTAINER_TO_STRING +#define VS_PRINT_CONTAINER_2_STRING PRINT_CONTAINER_2_STRING +#define VS_PRINT_VECTOR_TO_STRING PRINT_VECTOR_TO_STRING +#define VS_PRINT_VECTOR_2_STRING PRINT_VECTOR_2_STRING +#define VS_PRINT_VECTOR_MAPPING PRINT_VECTOR_MAPPING + +#else // VELOX_SHUFFLE_WRITER_PRINT + +#define VsPrint(...) // NOLINT +#define VsPrintLF(...) // NOLINT +#define VsPrintSplit(...) // NOLINT +#define VsPrintSplitLF(...) // NOLINT +#define VsPrintVectorRange(...) // NOLINT +#define VS_PRINT(a) +#define VS_PRINTLF(a) +#define VS_PRINT_FUNCTION_NAME() +#define VS_PRINT_FUNCTION_SPLIT_LINE() +#define VS_PRINT_CONTAINER(c) +#define VS_PRINT_CONTAINER_TO_STRING(c) +#define VS_PRINT_CONTAINER_2_STRING(c) +#define VS_PRINT_VECTOR_TO_STRING(v) +#define VS_PRINT_VECTOR_2_STRING(v) +#define VS_PRINT_VECTOR_MAPPING(v) + +#endif // end of VELOX_SHUFFLE_WRITER_PRINT + +enum SplitState { kInit, kPreAlloc, kSplit, kStop }; + +struct BinaryArrayResizeState { + bool inResize; + uint32_t partitionId; + uint32_t binaryIdx; + + BinaryArrayResizeState() : inResize(false) {} + BinaryArrayResizeState(uint32_t partitionId, uint32_t binaryIdx) + : inResize(false), partitionId(partitionId), binaryIdx(binaryIdx) {} +}; + +class VeloxHashBasedShuffleWriter : public VeloxShuffleWriter { + enum { + kValidityBufferIndex = 0, + kFixedWidthValueBufferIndex = 1, + kBinaryValueBufferIndex = 2, + kBinaryLengthBufferIndex = kFixedWidthValueBufferIndex + }; + + public: + struct BinaryBuf { + BinaryBuf(uint8_t* value, uint8_t* length, uint64_t valueCapacityIn, uint64_t valueOffsetIn) + : valuePtr(value), lengthPtr(length), valueCapacity(valueCapacityIn), valueOffset(valueOffsetIn) {} + + BinaryBuf(uint8_t* value, uint8_t* length, uint64_t valueCapacity) : BinaryBuf(value, length, valueCapacity, 0) {} + + BinaryBuf() : BinaryBuf(nullptr, nullptr, 0) {} + + uint8_t* valuePtr; + uint8_t* lengthPtr; + uint64_t valueCapacity; + uint64_t valueOffset; + }; + + static arrow::Result> create( + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* arrowPool); + + arrow::Status write(std::shared_ptr cb, int64_t memLimit) override; + + arrow::Status stop() override; + + arrow::Status reclaimFixedSize(int64_t size, int64_t* actual) override; + + const uint64_t cachedPayloadSize() const override; + + arrow::Status evictPartitionBuffers(uint32_t partitionId, bool reuseBuffers) override; + + // For test only. + void setPartitionBufferSize(uint32_t newSize) override; + + // for debugging + void printColumnsInfo() const { + VS_PRINT_FUNCTION_SPLIT_LINE(); + VS_PRINTLF(fixed_width_column_count_); + + VS_PRINT_CONTAINER(simple_column_indices_); + VS_PRINT_CONTAINER(binary_column_indices_); + VS_PRINT_CONTAINER(complex_column_indices_); + + VS_PRINT_VECTOR_2_STRING(velox_column_types_); + VS_PRINT_VECTOR_TO_STRING(arrow_column_types_); + } + + void printPartition() const { + VS_PRINT_FUNCTION_SPLIT_LINE(); + // row ID -> partition ID + VS_PRINT_VECTOR_MAPPING(row_2_partition_); + + // partition -> row count + VS_PRINT_VECTOR_MAPPING(partition_2_row_count_); + } + + void printPartitionBuffer() const { + VS_PRINT_FUNCTION_SPLIT_LINE(); + VS_PRINT_VECTOR_MAPPING(partition_2_buffer_size_); + VS_PRINT_VECTOR_MAPPING(partitionBufferBase_); + } + + void printPartition2Row() const { + VS_PRINT_FUNCTION_SPLIT_LINE(); + VS_PRINT_VECTOR_MAPPING(partition2RowOffsetBase_); + +#if VELOX_SHUFFLE_WRITER_PRINT + for (auto pid = 0; pid < numPartitions_; ++pid) { + auto begin = partition2RowOffsetBase_[pid]; + auto end = partition2RowOffsetBase_[pid + 1]; + VsPrint("partition", pid); + VsPrintVectorRange(rowOffset2RowId_, begin, end); + } +#endif + } + + void printInputHasNull() const { + VS_PRINT_FUNCTION_SPLIT_LINE(); + VS_PRINT_CONTAINER(input_has_null_); + } + + private: + VeloxHashBasedShuffleWriter( + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* pool) + : VeloxShuffleWriter(numPartitions, std::move(partitionWriter), std::move(options), std::move(veloxPool), pool) {} + + arrow::Status init(); + + arrow::Status initPartitions(); + + arrow::Status initColumnTypes(const facebook::velox::RowVector& rv); + + arrow::Status splitRowVector(const facebook::velox::RowVector& rv); + + arrow::Status initFromRowVector(const facebook::velox::RowVector& rv); + + arrow::Status buildPartition2Row(uint32_t rowNum); + + arrow::Status updateInputHasNull(const facebook::velox::RowVector& rv); + + void setSplitState(SplitState state); + + arrow::Status doSplit(const facebook::velox::RowVector& rv, int64_t memLimit); + + bool beyondThreshold(uint32_t partitionId, uint32_t newSize); + + uint32_t calculatePartitionBufferSize(const facebook::velox::RowVector& rv, int64_t memLimit); + + arrow::Status preAllocPartitionBuffers(uint32_t preAllocBufferSize); + + arrow::Status updateValidityBuffers(uint32_t partitionId, uint32_t newSize); + + arrow::Result> + allocateValidityBuffer(uint32_t col, uint32_t partitionId, uint32_t newSize); + + arrow::Status allocatePartitionBuffer(uint32_t partitionId, uint32_t newSize); + + arrow::Status splitFixedWidthValueBuffer(const facebook::velox::RowVector& rv); + + arrow::Status splitBoolType(const uint8_t* srcAddr, const std::vector& dstAddrs); + + arrow::Status splitValidityBuffer(const facebook::velox::RowVector& rv); + + arrow::Status splitBinaryArray(const facebook::velox::RowVector& rv); + + arrow::Status splitComplexType(const facebook::velox::RowVector& rv); + + arrow::Status evictBuffers( + uint32_t partitionId, + uint32_t numRows, + std::vector> buffers, + bool reuseBuffers); + + arrow::Result>> assembleBuffers(uint32_t partitionId, bool reuseBuffers); + + template + arrow::Status splitFixedType(const uint8_t* srcAddr, const std::vector& dstAddrs) { + for (auto& pid : partitionUsed_) { + auto dstPidBase = (T*)(dstAddrs[pid] + partitionBufferBase_[pid] * sizeof(T)); + auto pos = partition2RowOffsetBase_[pid]; + auto end = partition2RowOffsetBase_[pid + 1]; + for (; pos < end; ++pos) { + auto rowId = rowOffset2RowId_[pos]; + *dstPidBase++ = reinterpret_cast(srcAddr)[rowId]; // copy + } + } + return arrow::Status::OK(); + } + + arrow::Status splitBinaryType( + uint32_t binaryIdx, + const facebook::velox::FlatVector& src, + std::vector& dst); + + arrow::Result evictCachedPayload(int64_t size); + + arrow::Result> generateComplexTypeBuffers(facebook::velox::RowVectorPtr vector); + + arrow::Status resetValidityBuffer(uint32_t partitionId); + + arrow::Result shrinkPartitionBuffersMinSize(int64_t size); + + arrow::Result evictPartitionBuffersMinSize(int64_t size); + + arrow::Status shrinkPartitionBuffer(uint32_t partitionId); + + arrow::Status resetPartitionBuffer(uint32_t partitionId); + + // Resize the partition buffer to newSize. If preserveData is true, it will keep the data in buffer. + // Note when preserveData is false, and newSize is larger, this function can introduce unnecessary memory copy. + // In this case, use allocatePartitionBuffer to free current buffers and allocate new buffers instead. + arrow::Status resizePartitionBuffer(uint32_t partitionId, uint32_t newSize, bool preserveData); + + uint64_t valueBufferSizeForBinaryArray(uint32_t binaryIdx, uint32_t newSize); + + uint64_t valueBufferSizeForFixedWidthArray(uint32_t fixedWidthIndex, uint32_t newSize); + + void calculateSimpleColumnBytes(); + + void stat() const; + + bool shrinkPartitionBuffersAfterSpill() const; + + bool evictPartitionBuffersAfterSpill() const; + + arrow::Result partitionBufferSizeAfterShrink(uint32_t partitionId) const; + + bool isExtremelyLargeBatch(facebook::velox::RowVectorPtr& rv) const; + + arrow::Status partitioningAndDoSplit(facebook::velox::RowVectorPtr rv, int64_t memLimit); + + BinaryArrayResizeState binaryArrayResizeState_{}; + + bool hasComplexType_ = false; + std::vector isValidityBuffer_; + + // Store arrow column types. Calculated once. + std::vector> arrowColumnTypes_; + + // Store velox column types. Calculated once. + std::vector> veloxColumnTypes_; + + // How many fixed-width columns in the schema. Calculated once. + uint32_t fixedWidthColumnCount_ = 0; + + // The column indices of all binary types in the schema. + std::vector binaryColumnIndices_; + + // The column indices of all fixed-width and binary columns in the schema. + std::vector simpleColumnIndices_; + + // The column indices of all complex types in the schema, including Struct, Map, List columns. + std::vector complexColumnIndices_; + + // Total bytes of fixed-width buffers of all simple columns. Including validity buffers, value buffers of + // fixed-width types and length buffers of binary types. + // Used for estimating pre-allocated partition buffer size. Calculated once. + uint32_t fixedWidthBufferBytes_ = 0; + + // Used for calculating the average binary length. + // Updated for each input RowVector. + uint64_t totalInputNumRows_ = 0; + std::vector binaryArrayTotalSizeBytes_; + + // True if input column has null in any processed input RowVector. + // In the order of fixed-width columns + binary columns. + std::vector inputHasNull_; + + // Records which partitions are actually occurred in the current input RowVector. + // Most of the loops can loop on this array to avoid visiting unused partition id. + std::vector partitionUsed_; + + // Row ID -> Partition ID + // subscript: The index of row in the current input RowVector + // value: Partition ID + // Updated for each input RowVector. + std::vector row2Partition_; + + // Partition ID -> Row Count + // subscript: Partition ID + // value: How many rows does this partition have in the current input RowVector + // Updated for each input RowVector. + std::vector partition2RowCount_; + + // Note: partition2RowOffsetBase_ and rowOffset2RowId_ are the optimization of flattening the 2-dimensional vector + // into single dimension. + // The first dimension is the partition id. The second dimension is the ith occurrence of this partition in the + // input RowVector. The value is the index of the row in the input RowVector. + // partition2RowOffsetBase_ records the offset of the first dimension. + // + // The index of the ith occurrence of a give partition `pid` in the input RowVector can be calculated via + // rowOffset2RowId_[partition2RowOffsetBase_[pid] + i] + // i is in the range of [0, partition2RowCount_[pid]) + + // Partition ID -> Row offset, elements num: Partition num + 1 + // subscript: Partition ID + // value: The base row offset of this Partition + // Updated for each input RowVector. + std::vector partition2RowOffsetBase_; + + // Row offset -> Source row ID, elements num: input RowVector row num + // subscript: Row offset + // value: The index of row in the current input RowVector + // Updated for each input RowVector. + std::vector rowOffset2RowId_; + + // Partition buffers are used for holding the intermediate data during split. + // Partition ID -> Partition buffer size(unit is row) + std::vector partitionBufferSize_; + + // The write position of partition buffer. Updated after split. Reset when partition buffers are reallocated. + std::vector partitionBufferBase_; + + // Used by all simple types. Stores raw pointers of partition buffers. + std::vector> partitionValidityAddrs_; + // Used by fixed-width types. Stores raw pointers of partition buffers. + std::vector> partitionFixedWidthValueAddrs_; + // Used by binary types. Stores raw pointers and metadata of partition buffers. + std::vector> partitionBinaryAddrs_; + + // Used by complex types. + // Partition id -> Serialized complex data. + std::vector> complexTypeData_; + std::vector> complexTypeFlushBuffer_; + std::shared_ptr complexWriteType_; + + facebook::velox::serializer::presto::PrestoVectorSerde serde_; + + SplitState splitState_{kInit}; +}; // class VeloxHashBasedShuffleWriter + +} // namespace gluten diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc b/cpp/velox/shuffle/VeloxShuffleReader.cc index 5c4e0193681e..22298ef91b32 100644 --- a/cpp/velox/shuffle/VeloxShuffleReader.cc +++ b/cpp/velox/shuffle/VeloxShuffleReader.cc @@ -370,39 +370,104 @@ std::shared_ptr VeloxColumnarBatchDeserializer::next() { VeloxColumnarBatchDeserializerFactory::VeloxColumnarBatchDeserializerFactory( const std::shared_ptr& schema, const std::shared_ptr& codec, + const facebook::velox::common::CompressionKind veloxCompressionType, const RowTypePtr& rowType, int32_t batchSize, arrow::MemoryPool* memoryPool, - std::shared_ptr veloxPool) + std::shared_ptr veloxPool, + ShuffleWriterType shuffleWriterType) : schema_(schema), codec_(codec), + veloxCompressionType_(veloxCompressionType), rowType_(rowType), batchSize_(batchSize), memoryPool_(memoryPool), - veloxPool_(veloxPool) { + veloxPool_(veloxPool), + shuffleWriterType_(shuffleWriterType) { initFromSchema(); } std::unique_ptr VeloxColumnarBatchDeserializerFactory::createDeserializer( std::shared_ptr in) { - return std::make_unique( - std::move(in), - schema_, - codec_, + if (shuffleWriterType_ == kHashShuffle) { + return std::make_unique( + std::move(in), + schema_, + codec_, + rowType_, + batchSize_, + memoryPool_, + veloxPool_.get(), + &isValidityBuffer_, + hasComplexType_, + deserializeTime_, + decompressTime_); + } + return std::make_unique( + veloxPool_, rowType_, batchSize_, - memoryPool_, - veloxPool_.get(), - &isValidityBuffer_, - hasComplexType_, - deserializeTime_, - decompressTime_); + veloxCompressionType_, + [this](int64_t decompressionTime) { this->decompressTime_ += decompressionTime; }, + [this](int64_t deserializeTime) { this->deserializeTime_ += deserializeTime; }, + in); +} + +VeloxShuffleReaderOutStreamWrapper::VeloxShuffleReaderOutStreamWrapper( + const std::shared_ptr& veloxPool, + const RowTypePtr& rowType, + int32_t batchSize, + facebook::velox::common::CompressionKind veloxCompressionType, + const std::function decompressionTimeAccumulator, + const std::function deserializeTimeAccumulator, + const std::shared_ptr in) + : veloxPool_(veloxPool), + rowType_(rowType), + batchSize_(batchSize), + veloxCompressionType_(veloxCompressionType), + decompressionTimeAccumulator_(decompressionTimeAccumulator), + deserializeTimeAccumulator_(deserializeTimeAccumulator) { + constexpr uint64_t kMaxReadBufferSize = (1 << 20) - AlignedBuffer::kPaddedSize; + auto buffer = AlignedBuffer::allocate(kMaxReadBufferSize, veloxPool_.get()); + in_ = std::make_unique(std::move(in), std::move(buffer)); + serdeOptions_ = {false, veloxCompressionType_}; + RowVectorPtr rowVector; +} + +std::shared_ptr VeloxShuffleReaderOutStreamWrapper::next() { + if (!in_->hasNext()) { + return nullptr; + } + + RowVectorPtr rowVector; + VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_, &rowVector, &serdeOptions_); + + if (rowVector->size() >= batchSize_) { + return std::make_shared(std::move(rowVector)); + } + + while (rowVector->size() < batchSize_ && in_->hasNext()) { + RowVectorPtr rowVectorTemp; + VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_, &rowVectorTemp, &serdeOptions_); + rowVector->append(rowVectorTemp.get()); + } + + int64_t decompressTime = 0LL; + int64_t deserializeTime = 0LL; + + decompressionTimeAccumulator_(decompressTime); + deserializeTimeAccumulator_(deserializeTime); + return std::make_shared(std::move(rowVector)); } arrow::MemoryPool* VeloxColumnarBatchDeserializerFactory::getPool() { return memoryPool_; } +ShuffleWriterType VeloxColumnarBatchDeserializerFactory::getShuffleWriterType() { + return shuffleWriterType_; +} + int64_t VeloxColumnarBatchDeserializerFactory::getDecompressTime() { return decompressTime_; } @@ -440,4 +505,30 @@ void VeloxColumnarBatchDeserializerFactory::initFromSchema() { } } } + +VeloxInputStream::VeloxInputStream(std::shared_ptr input, facebook::velox::BufferPtr buffer) + : in_(std::move(input)), buffer_(std::move(buffer)) { + next(true); +} + +bool VeloxInputStream::hasNext() { + if (offset_ == 0) { + return false; + } + if (ranges()[0].position >= ranges()[0].size) { + next(true); + return offset_ != 0; + } + return true; +} + +void VeloxInputStream::next(bool throwIfPastEnd) { + const uint32_t readBytes = buffer_->capacity(); + offset_ = in_->Read(readBytes, buffer_->asMutable()).ValueOr(0); + if (offset_ > 0) { + int32_t realBytes = offset_; + VELOX_CHECK_LT(0, realBytes, "Reading past end of spill file"); + setRange({buffer_->asMutable(), realBytes, 0}); + } +} } // namespace gluten diff --git a/cpp/velox/shuffle/VeloxShuffleReader.h b/cpp/velox/shuffle/VeloxShuffleReader.h index 18df38006e96..3a0d8f9ffb73 100644 --- a/cpp/velox/shuffle/VeloxShuffleReader.h +++ b/cpp/velox/shuffle/VeloxShuffleReader.h @@ -22,6 +22,8 @@ #include "velox/type/Type.h" #include "velox/vector/ComplexVector.h" +#include + namespace gluten { class VeloxColumnarBatchDeserializer final : public ColumnarBatchIterator { @@ -59,15 +61,57 @@ class VeloxColumnarBatchDeserializer final : public ColumnarBatchIterator { bool reachEos_{false}; }; +class VeloxInputStream : public facebook::velox::ByteInputStream { + public: + VeloxInputStream(std::shared_ptr input, facebook::velox::BufferPtr buffer); + + bool hasNext(); + + void next(bool throwIfPastEnd) override; + + std::shared_ptr in_; + const facebook::velox::BufferPtr buffer_; + uint64_t offset_ = -1; +}; + +class VeloxShuffleReaderOutStreamWrapper : public ColumnarBatchIterator { + public: + VeloxShuffleReaderOutStreamWrapper( + const std::shared_ptr& veloxPool, + const facebook::velox::RowTypePtr& rowType, + int32_t batchSize, + const facebook::velox::common::CompressionKind veloxCompressionType, + const std::function decompressionTimeAccumulator, + const std::function deserializeTimeAccumulator, + const std::shared_ptr in); + + std::shared_ptr next(); + + private: + std::shared_ptr veloxPool_; + facebook::velox::RowTypePtr rowType_; + std::vector batches_; + bool reachEos_{false}; + int32_t rowCount_; + int32_t batchSize_; + facebook::velox::common::CompressionKind veloxCompressionType_; + facebook::velox::serializer::presto::PrestoVectorSerde::PrestoOptions serdeOptions_; + std::function decompressionTimeAccumulator_; + std::function deserializeTimeAccumulator_; + std::shared_ptr in_; +}; + class VeloxColumnarBatchDeserializerFactory : public DeserializerFactory { public: VeloxColumnarBatchDeserializerFactory( const std::shared_ptr& schema, const std::shared_ptr& codec, + const facebook::velox::common::CompressionKind veloxCompressionType, const facebook::velox::RowTypePtr& rowType, int32_t batchSize, arrow::MemoryPool* memoryPool, - std::shared_ptr veloxPool); + std::shared_ptr veloxPool, + ShuffleWriterType shuffleWriterType); std::unique_ptr createDeserializer(std::shared_ptr in) override; @@ -77,9 +121,12 @@ class VeloxColumnarBatchDeserializerFactory : public DeserializerFactory { int64_t getDeserializeTime() override; + ShuffleWriterType getShuffleWriterType() override; + private: std::shared_ptr schema_; std::shared_ptr codec_; + facebook::velox::common::CompressionKind veloxCompressionType_; facebook::velox::RowTypePtr rowType_; int32_t batchSize_; arrow::MemoryPool* memoryPool_; @@ -88,6 +135,8 @@ class VeloxColumnarBatchDeserializerFactory : public DeserializerFactory { std::vector isValidityBuffer_; bool hasComplexType_{false}; + ShuffleWriterType shuffleWriterType_; + int64_t deserializeTime_{0}; int64_t decompressTime_{0}; diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.h b/cpp/velox/shuffle/VeloxShuffleWriter.h index e699a323bb4b..104b87616291 100644 --- a/cpp/velox/shuffle/VeloxShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxShuffleWriter.h @@ -46,157 +46,60 @@ namespace gluten { -// set 1 to open print -#define VELOX_SHUFFLE_WRITER_PRINT 0 - -#if VELOX_SHUFFLE_WRITER_PRINT - -#define VsPrint Print -#define VsPrintLF PrintLF -#define VsPrintSplit PrintSplit -#define VsPrintSplitLF PrintSplitLF -#define VsPrintVectorRange PrintVectorRange -#define VS_PRINT PRINT -#define VS_PRINTLF PRINTLF -#define VS_PRINT_FUNCTION_NAME PRINT_FUNCTION_NAME -#define VS_PRINT_FUNCTION_SPLIT_LINE PRINT_FUNCTION_SPLIT_LINE -#define VS_PRINT_CONTAINER PRINT_CONTAINER -#define VS_PRINT_CONTAINER_TO_STRING PRINT_CONTAINER_TO_STRING -#define VS_PRINT_CONTAINER_2_STRING PRINT_CONTAINER_2_STRING -#define VS_PRINT_VECTOR_TO_STRING PRINT_VECTOR_TO_STRING -#define VS_PRINT_VECTOR_2_STRING PRINT_VECTOR_2_STRING -#define VS_PRINT_VECTOR_MAPPING PRINT_VECTOR_MAPPING - -#else // VELOX_SHUFFLE_WRITER_PRINT - -#define VsPrint(...) // NOLINT -#define VsPrintLF(...) // NOLINT -#define VsPrintSplit(...) // NOLINT -#define VsPrintSplitLF(...) // NOLINT -#define VsPrintVectorRange(...) // NOLINT -#define VS_PRINT(a) -#define VS_PRINTLF(a) -#define VS_PRINT_FUNCTION_NAME() -#define VS_PRINT_FUNCTION_SPLIT_LINE() -#define VS_PRINT_CONTAINER(c) -#define VS_PRINT_CONTAINER_TO_STRING(c) -#define VS_PRINT_CONTAINER_2_STRING(c) -#define VS_PRINT_VECTOR_TO_STRING(v) -#define VS_PRINT_VECTOR_2_STRING(v) -#define VS_PRINT_VECTOR_MAPPING(v) - -#endif // end of VELOX_SHUFFLE_WRITER_PRINT - -enum SplitState { kInit, kPreAlloc, kSplit, kStop }; -enum EvictState { kEvictable, kUnevictable }; - -struct BinaryArrayResizeState { - bool inResize; - uint32_t partitionId; - uint32_t binaryIdx; - - BinaryArrayResizeState() : inResize(false) {} - BinaryArrayResizeState(uint32_t partitionId, uint32_t binaryIdx) - : inResize(false), partitionId(partitionId), binaryIdx(binaryIdx) {} -}; - -class VeloxShuffleWriter final : public ShuffleWriter { - enum { - kValidityBufferIndex = 0, - kFixedWidthValueBufferIndex = 1, - kBinaryValueBufferIndex = 2, - kBinaryLengthBufferIndex = kFixedWidthValueBufferIndex - }; - +class VeloxShuffleWriter : public ShuffleWriter { public: - struct BinaryBuf { - BinaryBuf(uint8_t* value, uint8_t* length, uint64_t valueCapacityIn, uint64_t valueOffsetIn) - : valuePtr(value), lengthPtr(length), valueCapacity(valueCapacityIn), valueOffset(valueOffsetIn) {} - - BinaryBuf(uint8_t* value, uint8_t* length, uint64_t valueCapacity) : BinaryBuf(value, length, valueCapacity, 0) {} - - BinaryBuf() : BinaryBuf(nullptr, nullptr, 0) {} - - uint8_t* valuePtr; - uint8_t* lengthPtr; - uint64_t valueCapacity; - uint64_t valueOffset; - }; - - static arrow::Result> create( - uint32_t numPartitions, - std::unique_ptr partitionWriter, - ShuffleWriterOptions options, - std::shared_ptr veloxPool, - arrow::MemoryPool* arrowPool); - - arrow::Status split(std::shared_ptr cb, int64_t memLimit) override; - - arrow::Status stop() override; - - arrow::Status reclaimFixedSize(int64_t size, int64_t* actual) override; - - const uint64_t cachedPayloadSize() const override; - - arrow::Status evictPartitionBuffers(uint32_t partitionId, bool reuseBuffers); - - int64_t rawPartitionBytes() const; - - // For test only. - void setPartitionBufferSize(uint32_t newSize); + facebook::velox::RowVectorPtr getStrippedRowVector(const facebook::velox::RowVector& rv) { + // get new row type + auto rowType = rv.type()->asRow(); + auto typeChildren = rowType.children(); + typeChildren.erase(typeChildren.begin()); + auto newRowType = facebook::velox::ROW(std::move(typeChildren)); + + // get length + auto length = rv.size(); + + // get children + auto children = rv.children(); + children.erase(children.begin()); + + return std::make_shared( + rv.pool(), newRowType, facebook::velox::BufferPtr(nullptr), length, std::move(children)); + } - // for debugging - void printColumnsInfo() const { - VS_PRINT_FUNCTION_SPLIT_LINE(); - VS_PRINTLF(fixed_width_column_count_); + const int32_t* getFirstColumn(const facebook::velox::RowVector& rv) { + VELOX_CHECK(rv.childrenSize() > 0, "RowVector missing partition id column."); - VS_PRINT_CONTAINER(simple_column_indices_); - VS_PRINT_CONTAINER(binary_column_indices_); - VS_PRINT_CONTAINER(complex_column_indices_); + auto& firstChild = rv.childAt(0); + VELOX_CHECK(firstChild->isFlatEncoding(), "Partition id (field 0) is not flat encoding."); + VELOX_CHECK( + firstChild->type()->isInteger(), + "Partition id (field 0) should be integer, but got {}", + firstChild->type()->toString()); - VS_PRINT_VECTOR_2_STRING(velox_column_types_); - VS_PRINT_VECTOR_TO_STRING(arrow_column_types_); + // first column is partition key hash value or pid + return firstChild->asFlatVector()->rawValues(); } - void printPartition() const { - VS_PRINT_FUNCTION_SPLIT_LINE(); - // row ID -> partition ID - VS_PRINT_VECTOR_MAPPING(row_2_partition_); - - // partition -> row count - VS_PRINT_VECTOR_MAPPING(partition_2_row_count_); - } + // For test only. + virtual void setPartitionBufferSize(uint32_t newSize) {} - void printPartitionBuffer() const { - VS_PRINT_FUNCTION_SPLIT_LINE(); - VS_PRINT_VECTOR_MAPPING(partition_2_buffer_size_); - VS_PRINT_VECTOR_MAPPING(partitionBufferBase_); + virtual arrow::Status evictPartitionBuffers(uint32_t partitionId, bool reuseBuffers) { + return arrow::Status::OK(); } - void printPartition2Row() const { - VS_PRINT_FUNCTION_SPLIT_LINE(); - VS_PRINT_VECTOR_MAPPING(partition2RowOffsetBase_); - -#if VELOX_SHUFFLE_WRITER_PRINT - for (auto pid = 0; pid < numPartitions_; ++pid) { - auto begin = partition2RowOffsetBase_[pid]; - auto end = partition2RowOffsetBase_[pid + 1]; - VsPrint("partition", pid); - VsPrintVectorRange(rowOffset2RowId_, begin, end); - } -#endif + virtual arrow::Status evictRowVector(uint32_t partitionId) { + return arrow::Status::OK(); } - void printInputHasNull() const { - VS_PRINT_FUNCTION_SPLIT_LINE(); - VS_PRINT_CONTAINER(input_has_null_); + virtual const uint64_t cachedPayloadSize() const { + return 0; } int32_t maxBatchSize() const { return maxBatchSize_; } - private: + protected: VeloxShuffleWriter( uint32_t numPartitions, std::unique_ptr partitionWriter, @@ -209,216 +112,19 @@ class VeloxShuffleWriter final : public ShuffleWriter { serdeOptions_.useLosslessTimestamp = true; } - arrow::Status init(); - - arrow::Status initPartitions(); - - arrow::Status initColumnTypes(const facebook::velox::RowVector& rv); - - arrow::Status splitRowVector(const facebook::velox::RowVector& rv); - - arrow::Status initFromRowVector(const facebook::velox::RowVector& rv); - - arrow::Status buildPartition2Row(uint32_t rowNum); - - arrow::Status updateInputHasNull(const facebook::velox::RowVector& rv); - - void setSplitState(SplitState state); - - arrow::Status doSplit(const facebook::velox::RowVector& rv, int64_t memLimit); - - bool beyondThreshold(uint32_t partitionId, uint32_t newSize); - - uint32_t calculatePartitionBufferSize(const facebook::velox::RowVector& rv, int64_t memLimit); - - arrow::Status preAllocPartitionBuffers(uint32_t preAllocBufferSize); - - arrow::Status updateValidityBuffers(uint32_t partitionId, uint32_t newSize); - - arrow::Result> - allocateValidityBuffer(uint32_t col, uint32_t partitionId, uint32_t newSize); - - arrow::Status allocatePartitionBuffer(uint32_t partitionId, uint32_t newSize); - - arrow::Status splitFixedWidthValueBuffer(const facebook::velox::RowVector& rv); - - arrow::Status splitBoolType(const uint8_t* srcAddr, const std::vector& dstAddrs); - - arrow::Status splitValidityBuffer(const facebook::velox::RowVector& rv); + virtual ~VeloxShuffleWriter() = default; - arrow::Status splitBinaryArray(const facebook::velox::RowVector& rv); - - arrow::Status splitComplexType(const facebook::velox::RowVector& rv); - - arrow::Status evictBuffers( - uint32_t partitionId, - uint32_t numRows, - std::vector> buffers, - bool reuseBuffers); - - arrow::Result>> assembleBuffers(uint32_t partitionId, bool reuseBuffers); - - template - arrow::Status splitFixedType(const uint8_t* srcAddr, const std::vector& dstAddrs) { - for (auto& pid : partitionUsed_) { - auto dstPidBase = (T*)(dstAddrs[pid] + partitionBufferBase_[pid] * sizeof(T)); - auto pos = partition2RowOffsetBase_[pid]; - auto end = partition2RowOffsetBase_[pid + 1]; - for (; pos < end; ++pos) { - auto rowId = rowOffset2RowId_[pos]; - *dstPidBase++ = reinterpret_cast(srcAddr)[rowId]; // copy - } - } - return arrow::Status::OK(); - } - - arrow::Status splitBinaryType( - uint32_t binaryIdx, - const facebook::velox::FlatVector& src, - std::vector& dst); - - arrow::Result evictCachedPayload(int64_t size); - - arrow::Result> generateComplexTypeBuffers(facebook::velox::RowVectorPtr vector); - - arrow::Status resetValidityBuffer(uint32_t partitionId); - - arrow::Result shrinkPartitionBuffersMinSize(int64_t size); - - arrow::Result evictPartitionBuffersMinSize(int64_t size); - - arrow::Status shrinkPartitionBuffer(uint32_t partitionId); - - arrow::Status resetPartitionBuffer(uint32_t partitionId); - - // Resize the partition buffer to newSize. If preserveData is true, it will keep the data in buffer. - // Note when preserveData is false, and newSize is larger, this function can introduce unnecessary memory copy. - // In this case, use allocatePartitionBuffer to free current buffers and allocate new buffers instead. - arrow::Status resizePartitionBuffer(uint32_t partitionId, uint32_t newSize, bool preserveData); - - uint64_t valueBufferSizeForBinaryArray(uint32_t binaryIdx, uint32_t newSize); - - uint64_t valueBufferSizeForFixedWidthArray(uint32_t fixedWidthIndex, uint32_t newSize); - - void calculateSimpleColumnBytes(); - - void stat() const; - - bool shrinkPartitionBuffersAfterSpill() const; - - bool evictPartitionBuffersAfterSpill() const; - - arrow::Result partitionBufferSizeAfterShrink(uint32_t partitionId) const; - - bool isExtremelyLargeBatch(facebook::velox::RowVectorPtr& rv) const; - - arrow::Status partitioningAndDoSplit(facebook::velox::RowVectorPtr rv, int64_t memLimit); - - SplitState splitState_{kInit}; + std::vector> arenas_; - EvictState evictState_{kEvictable}; + facebook::velox::serializer::presto::PrestoVectorSerde::PrestoOptions serdeOptions_; - BinaryArrayResizeState binaryArrayResizeState_{}; + std::shared_ptr veloxPool_; bool supportAvx512_ = false; - bool hasComplexType_ = false; - std::vector isValidityBuffer_; - - // Store arrow column types. Calculated once. - std::vector> arrowColumnTypes_; - - // Store velox column types. Calculated once. - std::vector> veloxColumnTypes_; - - // How many fixed-width columns in the schema. Calculated once. - uint32_t fixedWidthColumnCount_ = 0; - - // The column indices of all binary types in the schema. - std::vector binaryColumnIndices_; - - // The column indices of all fixed-width and binary columns in the schema. - std::vector simpleColumnIndices_; - - // The column indices of all complex types in the schema, including Struct, Map, List columns. - std::vector complexColumnIndices_; - - // Total bytes of fixed-width buffers of all simple columns. Including validity buffers, value buffers of - // fixed-width types and length buffers of binary types. - // Used for estimating pre-allocated partition buffer size. Calculated once. - uint32_t fixedWidthBufferBytes_ = 0; - - // Used for calculating the average binary length. - // Updated for each input RowVector. - uint64_t totalInputNumRows_ = 0; - std::vector binaryArrayTotalSizeBytes_; - - // True if input column has null in any processed input RowVector. - // In the order of fixed-width columns + binary columns. - std::vector inputHasNull_; - - // Records which partitions are actually occurred in the current input RowVector. - // Most of the loops can loop on this array to avoid visiting unused partition id. - std::vector partitionUsed_; - - // Row ID -> Partition ID - // subscript: The index of row in the current input RowVector - // value: Partition ID - // Updated for each input RowVector. - std::vector row2Partition_; - - // Partition ID -> Row Count - // subscript: Partition ID - // value: How many rows does this partition have in the current input RowVector - // Updated for each input RowVector. - std::vector partition2RowCount_; - - // Note: partition2RowOffsetBase_ and rowOffset2RowId_ are the optimization of flattening the 2-dimensional vector - // into single dimension. - // The first dimension is the partition id. The second dimension is the ith occurrence of this partition in the - // input RowVector. The value is the index of the row in the input RowVector. - // partition2RowOffsetBase_ records the offset of the first dimension. - // - // The index of the ith occurrence of a give partition `pid` in the input RowVector can be calculated via - // rowOffset2RowId_[partition2RowOffsetBase_[pid] + i] - // i is in the range of [0, partition2RowCount_[pid]) - - // Partition ID -> Row offset, elements num: Partition num + 1 - // subscript: Partition ID - // value: The base row offset of this Partition - // Updated for each input RowVector. - std::vector partition2RowOffsetBase_; - - // Row offset -> Source row ID, elements num: input RowVector row num - // subscript: Row offset - // value: The index of row in the current input RowVector - // Updated for each input RowVector. - std::vector rowOffset2RowId_; - - // Partition buffers are used for holding the intermediate data during split. - // Partition ID -> Partition buffer size(unit is row) - std::vector partitionBufferSize_; - - // The write position of partition buffer. Updated after split. Reset when partition buffers are reallocated. - std::vector partitionBufferBase_; - - // Used by all simple types. Stores raw pointers of partition buffers. - std::vector> partitionValidityAddrs_; - // Used by fixed-width types. Stores raw pointers of partition buffers. - std::vector> partitionFixedWidthValueAddrs_; - // Used by binary types. Stores raw pointers and metadata of partition buffers. - std::vector> partitionBinaryAddrs_; - - // Used by complex types. - // Partition id -> Serialized complex data. - std::vector> complexTypeData_; - std::vector> complexTypeFlushBuffer_; - std::shared_ptr complexWriteType_; + int32_t maxBatchSize_{0}; - std::shared_ptr veloxPool_; - std::vector> arenas_; - facebook::velox::serializer::presto::PrestoVectorSerde serde_; - facebook::velox::serializer::presto::PrestoVectorSerde::PrestoOptions serdeOptions_; + enum EvictState { kEvictable, kUnevictable }; // stat enum CpuWallTimingType { @@ -474,7 +180,28 @@ class VeloxShuffleWriter final : public ShuffleWriter { } facebook::velox::CpuWallTiming cpuWallTimingList_[CpuWallTimingNum]; - int32_t maxBatchSize_{0}; -}; // class VeloxShuffleWriter + + EvictState evictState_{kEvictable}; + + class EvictGuard { + public: + explicit EvictGuard(EvictState& evictState) : evictState_(evictState) { + evictState_ = EvictState::kUnevictable; + } + + ~EvictGuard() { + evictState_ = EvictState::kEvictable; + } + + // For safety and clarity. + EvictGuard(const EvictGuard&) = delete; + EvictGuard& operator=(const EvictGuard&) = delete; + EvictGuard(EvictGuard&&) = delete; + EvictGuard& operator=(EvictGuard&&) = delete; + + private: + EvictState& evictState_; + }; +}; } // namespace gluten diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc new file mode 100644 index 000000000000..b0c2cc8adc7b --- /dev/null +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "VeloxSortBasedShuffleWriter.h" +#include "memory/ArrowMemory.h" +#include "memory/VeloxColumnarBatch.h" +#include "memory/VeloxMemoryManager.h" +#include "shuffle/ShuffleSchema.h" +#include "utils/Common.h" +#include "utils/VeloxArrowUtils.h" +#include "utils/macros.h" +#include "velox/common/base/Nulls.h" +#include "velox/type/Type.h" +#include "velox/vector/ComplexVector.h" + +#if defined(__x86_64__) +#include +#include +#elif defined(__aarch64__) +#include +#endif + +namespace gluten { + +#define VELOX_SHUFFLE_WRITER_LOG_FLAG 0 + +// macro to rotate left an 8-bit value 'x' given the shift 's' is a 32-bit integer +// (x is left shifted by 's' modulo 8) OR (x right shifted by (8 - 's' modulo 8)) +#if !defined(__x86_64__) +#define rotateLeft(x, s) (x << (s - ((s >> 3) << 3)) | x >> (8 - (s - ((s >> 3) << 3)))) +#endif + +// on x86 machines, _MM_HINT_T0,T1,T2 are defined as 1, 2, 3 +// equivalent mapping to __builtin_prefetch hints is 3, 2, 1 +#if defined(__x86_64__) +#define PREFETCHT0(ptr) _mm_prefetch(ptr, _MM_HINT_T0) +#define PREFETCHT1(ptr) _mm_prefetch(ptr, _MM_HINT_T1) +#define PREFETCHT2(ptr) _mm_prefetch(ptr, _MM_HINT_T2) +#else +#define PREFETCHT0(ptr) __builtin_prefetch(ptr, 0, 3) +#define PREFETCHT1(ptr) __builtin_prefetch(ptr, 0, 2) +#define PREFETCHT2(ptr) __builtin_prefetch(ptr, 0, 1) +#endif + +arrow::Result> VeloxSortBasedShuffleWriter::create( + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* arrowPool) { + std::shared_ptr res(new VeloxSortBasedShuffleWriter( + numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool)); + RETURN_NOT_OK(res->init()); + return res; +} // namespace gluten + +arrow::Status VeloxSortBasedShuffleWriter::init() { +#if defined(__x86_64__) + supportAvx512_ = __builtin_cpu_supports("avx512bw"); +#else + supportAvx512_ = false; +#endif + + ARROW_ASSIGN_OR_RAISE( + partitioner_, Partitioner::make(options_.partitioning, numPartitions_, options_.startPartitionId)); + DLOG(INFO) << "Create partitioning type: " << std::to_string(options_.partitioning); + + partition2RowCount_.resize(numPartitions_); + rowVectorIndexMap_.reserve(numPartitions_); + for (auto pid = 0; pid < numPartitions_; ++pid) { + rowVectorIndexMap_[pid].reserve(options_.bufferSize); + } + + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::doSort(facebook::velox::RowVectorPtr rv, int64_t memLimit) { + currentInputColumnBytes_ += rv->estimateFlatSize(); + batches_.push_back(rv); + if (currentInputColumnBytes_ > memLimit) { + for (auto pid = 0; pid < numPartitions(); ++pid) { + RETURN_NOT_OK(evictRowVector(pid)); + partition2RowCount_[pid] = 0; + } + batches_.clear(); + currentInputColumnBytes_ = 0; + } + setSortState(SortState::kSortInit); + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::write(std::shared_ptr cb, int64_t memLimit) { + if (options_.partitioning == Partitioning::kSingle) { + auto veloxColumnBatch = VeloxColumnarBatch::from(veloxPool_.get(), cb); + VELOX_CHECK_NOT_NULL(veloxColumnBatch); + auto rv = veloxColumnBatch->getFlattenedRowVector(); + RETURN_NOT_OK(initFromRowVector(*rv.get())); + RETURN_NOT_OK(doSort(rv, partitionWriter_.get()->options().sortBufferMaxSize)); + } else if (options_.partitioning == Partitioning::kRange) { + auto compositeBatch = std::dynamic_pointer_cast(cb); + VELOX_CHECK_NOT_NULL(compositeBatch); + auto batches = compositeBatch->getBatches(); + VELOX_CHECK_EQ(batches.size(), 2); + auto pidBatch = VeloxColumnarBatch::from(veloxPool_.get(), batches[0]); + auto pidArr = getFirstColumn(*(pidBatch->getRowVector())); + START_TIMING(cpuWallTimingList_[CpuWallTimingCompute]); + setSortState(SortState::kSort); + RETURN_NOT_OK(partitioner_->compute(pidArr, pidBatch->numRows(), batches_.size(), rowVectorIndexMap_)); + END_TIMING(); + auto rvBatch = VeloxColumnarBatch::from(veloxPool_.get(), batches[1]); + auto rv = rvBatch->getFlattenedRowVector(); + RETURN_NOT_OK(initFromRowVector(*rv.get())); + RETURN_NOT_OK(doSort(rv, partitionWriter_.get()->options().sortBufferMaxSize)); + } else { + auto veloxColumnBatch = VeloxColumnarBatch::from(veloxPool_.get(), cb); + VELOX_CHECK_NOT_NULL(veloxColumnBatch); + facebook::velox::RowVectorPtr rv; + START_TIMING(cpuWallTimingList_[CpuWallTimingFlattenRV]); + rv = veloxColumnBatch->getFlattenedRowVector(); + END_TIMING(); + if (partitioner_->hasPid()) { + auto pidArr = getFirstColumn(*rv); + START_TIMING(cpuWallTimingList_[CpuWallTimingCompute]); + setSortState(SortState::kSort); + RETURN_NOT_OK(partitioner_->compute(pidArr, rv->size(), batches_.size(), rowVectorIndexMap_)); + END_TIMING(); + auto strippedRv = getStrippedRowVector(*rv); + RETURN_NOT_OK(initFromRowVector(*strippedRv)); + RETURN_NOT_OK(doSort(strippedRv, partitionWriter_.get()->options().sortBufferMaxSize)); + } else { + RETURN_NOT_OK(initFromRowVector(*rv)); + START_TIMING(cpuWallTimingList_[CpuWallTimingCompute]); + setSortState(SortState::kSort); + RETURN_NOT_OK(partitioner_->compute(nullptr, rv->size(), batches_.size(), rowVectorIndexMap_)); + END_TIMING(); + RETURN_NOT_OK(doSort(rv, partitionWriter_.get()->options().sortBufferMaxSize)); + } + } + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::evictBatch( + uint32_t partitionId, + std::ostringstream* output, + facebook::velox::OStreamOutputStream* out, + facebook::velox::RowTypePtr* rowTypePtr) { + int64_t rawSize = batch_->size(); + batch_->flush(out); + const std::string& outputStr = output->str(); + RETURN_NOT_OK(partitionWriter_->evict(partitionId, rawSize, outputStr.c_str(), outputStr.size())); + batch_.reset(); + output->clear(); + output->str(""); + batch_ = std::make_unique(veloxPool_.get(), serde_.get()); + batch_->createStreamTree(*rowTypePtr, options_.bufferSize, &serdeOptions_); + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::evictRowVector(uint32_t partitionId) { + int32_t rowNum = 0; + const int32_t maxBatchNum = options_.bufferSize; + auto rowTypePtr = std::static_pointer_cast(rowType_.value()); + std::ostringstream output; + facebook::velox::OStreamOutputStream out(&output); + + if (options_.partitioning != Partitioning::kSingle) { + if (auto it = rowVectorIndexMap_.find(partitionId); it != rowVectorIndexMap_.end()) { + auto rowVectorIndex = it->second; + const int32_t outputSize = rowVectorIndex.size(); + + std::map> groupedIndices; + std::map groupedSize; + + int32_t tempVectorIndex = -1; + int32_t baseRowIndex = -1; + int32_t tempRowIndex = -1; + int32_t size = 1; + for (int start = 0; start < outputSize; start++) { + const int64_t rowVector = rowVectorIndex[start]; + const int32_t vectorIndex = static_cast(rowVector >> 32); + const int32_t rowIndex = static_cast(rowVector & 0xFFFFFFFFLL); + if (tempVectorIndex == -1) { + tempVectorIndex = vectorIndex; + baseRowIndex = rowIndex; + tempRowIndex = rowIndex; + } else { + if (vectorIndex == tempVectorIndex && rowIndex == tempRowIndex + 1) { + size += 1; + tempRowIndex = rowIndex; + } else { + groupedIndices[tempVectorIndex].push_back({baseRowIndex, size}); + groupedSize[tempVectorIndex] += size; + size = 1; + tempVectorIndex = vectorIndex; + baseRowIndex = rowIndex; + tempRowIndex = rowIndex; + } + } + } + groupedIndices[tempVectorIndex].push_back({baseRowIndex, size}); + groupedSize[tempVectorIndex] += size; + + for (auto& pair : groupedIndices) { + batch_->append(batches_[pair.first], pair.second); + rowNum += groupedSize[pair.first]; + if (rowNum >= maxBatchNum) { + rowNum = 0; + RETURN_NOT_OK(evictBatch(partitionId, &output, &out, &rowTypePtr)); + } + } + + rowVectorIndex.clear(); + rowVectorIndexMap_.erase(partitionId); + } + } else { + for (facebook::velox::RowVectorPtr rowVectorPtr : batches_) { + rowNum += rowVectorPtr->size(); + batch_->append(rowVectorPtr); + if (rowNum >= maxBatchNum) { + RETURN_NOT_OK(evictBatch(partitionId, &output, &out, &rowTypePtr)); + rowNum = 0; + } + } + } + if (rowNum > 0) { + RETURN_NOT_OK(evictBatch(partitionId, &output, &out, &rowTypePtr)); + } + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::stop() { + for (auto pid = 0; pid < numPartitions(); ++pid) { + RETURN_NOT_OK(evictRowVector(pid)); + partition2RowCount_[pid] = 0; + } + batches_.clear(); + currentInputColumnBytes_ = 0; + { + SCOPED_TIMER(cpuWallTimingList_[CpuWallTimingStop]); + setSortState(SortState::kSortStop); + RETURN_NOT_OK(partitionWriter_->stop(&metrics_)); + partitionBuffers_.clear(); + } + + stat(); + + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::initFromRowVector(const facebook::velox::RowVector& rv) { + if (!rowType_.has_value()) { + rowType_ = rv.type(); + serdeOptions_ = { + false, facebook::velox::common::stringToCompressionKind(partitionWriter_->options().compressionTypeStr)}; + batch_ = std::make_unique(veloxPool_.get(), serde_.get()); + batch_->createStreamTree( + std::static_pointer_cast(rowType_.value()), + options_.bufferSize, + &serdeOptions_); + } + return arrow::Status::OK(); +} + +arrow::Status VeloxSortBasedShuffleWriter::reclaimFixedSize(int64_t size, int64_t* actual) { + if (evictState_ == EvictState::kUnevictable) { + *actual = 0; + return arrow::Status::OK(); + } + EvictGuard evictGuard{evictState_}; + + if (sortState_ == SortState::kSortInit) { + for (auto pid = 0; pid < numPartitions(); ++pid) { + RETURN_NOT_OK(evictRowVector(pid)); + partition2RowCount_[pid] = 0; + } + batches_.clear(); + *actual = currentInputColumnBytes_; + currentInputColumnBytes_ = 0; + } + return arrow::Status::OK(); +} + +void VeloxSortBasedShuffleWriter::stat() const { +#if VELOX_SHUFFLE_WRITER_LOG_FLAG + for (int i = CpuWallTimingBegin; i != CpuWallTimingEnd; ++i) { + std::ostringstream oss; + auto& timing = cpuWallTimingList_[i]; + oss << "Velox shuffle writer stat:" << CpuWallTimingName((CpuWallTimingType)i); + oss << " " << timing.toString(); + if (timing.count > 0) { + oss << " wallNanos-avg:" << timing.wallNanos / timing.count; + oss << " cpuNanos-avg:" << timing.cpuNanos / timing.count; + } + LOG(INFO) << oss.str(); + } +#endif +} + +void VeloxSortBasedShuffleWriter::setSortState(SortState state) { + sortState_ = state; +} + +} // namespace gluten diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h new file mode 100644 index 000000000000..e3ac07dfcd82 --- /dev/null +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "velox/common/time/CpuWallTimer.h" +#include "velox/serializers/PrestoSerializer.h" +#include "velox/type/Type.h" +#include "velox/vector/ComplexVector.h" +#include "velox/vector/FlatVector.h" +#include "velox/vector/VectorStream.h" + +#include +#include +#include +#include +#include +#include + +#include "VeloxShuffleWriter.h" +#include "memory/VeloxMemoryManager.h" +#include "shuffle/PartitionWriter.h" +#include "shuffle/Partitioner.h" +#include "shuffle/Utils.h" + +#include "utils/Print.h" + +namespace gluten { + +enum SortState { kSortInit, kSort, kSortStop }; + +class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { + public: + static arrow::Result> create( + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* arrowPool); + + arrow::Status write(std::shared_ptr cb, int64_t memLimit) override; + + arrow::Status stop() override; + + arrow::Status reclaimFixedSize(int64_t size, int64_t* actual) override; + + arrow::Status evictRowVector(uint32_t partitionId) override; + + arrow::Status evictBatch( + uint32_t partitionId, + std::ostringstream* output, + facebook::velox::OStreamOutputStream* out, + facebook::velox::RowTypePtr* rowTypePtr); + + private: + VeloxSortBasedShuffleWriter( + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* pool) + : VeloxShuffleWriter(numPartitions, std::move(partitionWriter), std::move(options), std::move(veloxPool), pool) {} + + arrow::Status init(); + + arrow::Status initFromRowVector(const facebook::velox::RowVector& rv); + + void setSortState(SortState state); + + arrow::Status doSort(facebook::velox::RowVectorPtr rv, int64_t memLimit); + + void stat() const; + + std::optional rowType_; + + std::unique_ptr batch_; + + // Partition ID -> Row Count + // subscript: Partition ID + // value: How many rows does this partition have in the current input RowVector + // Updated for each input RowVector. + std::vector partition2RowCount_; + + std::unique_ptr serde_ = + std::make_unique(); + + std::vector batches_; + + std::unordered_map> rowVectorIndexMap_; + + std::unordered_map> rowVectorPartitionMap_; + + uint32_t currentInputColumnBytes_ = 0; + + SortState sortState_{kSortInit}; +}; // class VeloxSortBasedShuffleWriter + +} // namespace gluten diff --git a/cpp/velox/tests/VeloxShuffleWriterTest.cc b/cpp/velox/tests/VeloxShuffleWriterTest.cc index ffda945b1f60..fdf3e4491169 100644 --- a/cpp/velox/tests/VeloxShuffleWriterTest.cc +++ b/cpp/velox/tests/VeloxShuffleWriterTest.cc @@ -19,7 +19,9 @@ #include #include "shuffle/LocalPartitionWriter.h" +#include "shuffle/VeloxHashBasedShuffleWriter.h" #include "shuffle/VeloxShuffleWriter.h" +#include "shuffle/VeloxSortBasedShuffleWriter.h" #include "shuffle/rss/RssPartitionWriter.h" #include "utils/TestUtils.h" #include "utils/VeloxArrowUtils.h" @@ -68,12 +70,18 @@ std::vector createShuffleTestParams() { std::vector mergeBufferSizes = {0, 3, 4, 10, 4096}; for (const auto& compression : compressions) { + params.push_back(ShuffleTestParams{ShuffleWriterType::kSortShuffle, PartitionWriterType::kRss, compression, 0, 0}); for (const auto compressionThreshold : compressionThresholds) { for (const auto mergeBufferSize : mergeBufferSizes) { - params.push_back( - ShuffleTestParams{PartitionWriterType::kLocal, compression, compressionThreshold, mergeBufferSize}); + params.push_back(ShuffleTestParams{ + ShuffleWriterType::kHashShuffle, + PartitionWriterType::kLocal, + compression, + compressionThreshold, + mergeBufferSize}); } - params.push_back(ShuffleTestParams{PartitionWriterType::kRss, compression, compressionThreshold, 0}); + params.push_back(ShuffleTestParams{ + ShuffleWriterType::kHashShuffle, PartitionWriterType::kRss, compression, compressionThreshold, 0}); } } @@ -264,7 +272,9 @@ TEST_P(HashPartitioningShuffleWriter, hashLargeVectors) { auto shuffleWriter = createShuffleWriter(defaultArrowMemoryPool().get()); // calculate maxBatchSize_ ASSERT_NOT_OK(splitRowVector(*shuffleWriter, hashInputVector1_)); - VELOX_CHECK_EQ(shuffleWriter->maxBatchSize(), expectedMaxBatchSize); + if (GetParam().shuffleWriterType == kHashShuffle) { + VELOX_CHECK_EQ(shuffleWriter->maxBatchSize(), expectedMaxBatchSize); + } auto blockPid2 = takeRows({inputVector1_, inputVector2_, inputVector1_}, {{1, 2, 3, 4, 8}, {0, 1}, {1, 2, 3, 4, 8}}); auto blockPid1 = takeRows({inputVector1_}, {{0, 5, 6, 7, 9, 0, 5, 6, 7, 9}}); @@ -305,6 +315,9 @@ TEST_P(RoundRobinPartitioningShuffleWriter, roundRobin) { } TEST_P(RoundRobinPartitioningShuffleWriter, preAllocForceRealloc) { + if (GetParam().shuffleWriterType == kSortShuffle) { + return; + } ASSERT_NOT_OK(initShuffleWriterOptions()); shuffleWriterOptions_.bufferReallocThreshold = 0; // Force re-alloc on buffer size changed. auto shuffleWriter = createShuffleWriter(defaultArrowMemoryPool().get()); @@ -392,6 +405,9 @@ TEST_P(RoundRobinPartitioningShuffleWriter, preAllocForceReuse) { } TEST_P(RoundRobinPartitioningShuffleWriter, spillVerifyResult) { + if (GetParam().shuffleWriterType == kSortShuffle) { + return; + } ASSERT_NOT_OK(initShuffleWriterOptions()); auto shuffleWriter = createShuffleWriter(defaultArrowMemoryPool().get()); diff --git a/cpp/velox/utils/tests/LocalRssClient.h b/cpp/velox/utils/tests/LocalRssClient.h index 0033526bb3be..c5c1b5d2c359 100644 --- a/cpp/velox/utils/tests/LocalRssClient.h +++ b/cpp/velox/utils/tests/LocalRssClient.h @@ -30,7 +30,7 @@ class LocalRssClient : public RssClient { public: LocalRssClient(std::string dataFile) : dataFile_(dataFile) {} - int32_t pushPartitionData(int32_t partitionId, char* bytes, int64_t size) { + int32_t pushPartitionData(int32_t partitionId, const char* bytes, int64_t size) { auto idx = -1; auto maybeIdx = partitionIdx_.find(partitionId); if (maybeIdx == partitionIdx_.end()) { diff --git a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h index 972c0cb25850..66732c97a9a9 100644 --- a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h +++ b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h @@ -62,6 +62,7 @@ std::unique_ptr createPartitionWriter( } // namespace struct ShuffleTestParams { + ShuffleWriterType shuffleWriterType; PartitionWriterType partitionWriterType; arrow::Compression::type compressionType; int32_t compressionThreshold; @@ -69,8 +70,9 @@ struct ShuffleTestParams { std::string toString() const { std::ostringstream out; - out << "partitionWriterType = " << partitionWriterType << ", compressionType = " << compressionType - << ", compressionThreshold = " << compressionThreshold << ", mergeBufferSize = " << mergeBufferSize; + out << "shuffleWriterType = " << shuffleWriterType << ", partitionWriterType = " << partitionWriterType + << ", compressionType = " << compressionType << ", compressionThreshold = " << compressionThreshold + << ", mergeBufferSize = " << mergeBufferSize; return out.str(); } }; @@ -179,7 +181,7 @@ class VeloxShuffleWriterTestBase : public facebook::velox::test::VectorTestBase arrow::Status splitRowVector(VeloxShuffleWriter& shuffleWriter, facebook::velox::RowVectorPtr vector) { std::shared_ptr cb = std::make_shared(vector); - return shuffleWriter.split(cb, ShuffleWriter::kMinMemLimit); + return shuffleWriter.write(cb, ShuffleWriter::kMinMemLimit); } // Create multiple local dirs and join with comma. @@ -231,11 +233,47 @@ class VeloxShuffleWriterTest : public ::testing::TestWithParam createSpecificShuffleWriter( + arrow::MemoryPool* arrowPool, + std::unique_ptr partitionWriter, + ShuffleWriterOptions shuffleWriterOptions, + uint32_t numPartitions, + int32_t bufferSize) { + std::shared_ptr shuffleWriter; + if (GetParam().shuffleWriterType == kHashShuffle) { + shuffleWriterOptions.bufferSize = bufferSize; + GLUTEN_ASSIGN_OR_THROW( + shuffleWriter, + VeloxHashBasedShuffleWriter::create( + numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions), pool_, arrowPool)); + } else if ( + GetParam().shuffleWriterType == kSortShuffle && GetParam().partitionWriterType == PartitionWriterType::kRss) { + GLUTEN_ASSIGN_OR_THROW( + shuffleWriter, + VeloxSortBasedShuffleWriter::create( + numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions), pool_, arrowPool)); + } + return shuffleWriter; + } + protected: static void SetUpTestCase() { facebook::velox::memory::MemoryManager::testingSetInstance({}); @@ -276,9 +314,33 @@ class VeloxShuffleWriterTest : public ::testing::TestWithParam( - schema, std::move(codec), rowType, std::numeric_limits::max(), defaultArrowMemoryPool().get(), pool_); + schema, + std::move(codec), + veloxCompressionType, + rowType, + std::numeric_limits::max(), + defaultArrowMemoryPool().get(), + pool_, + GetParam().shuffleWriterType); auto reader = std::make_shared(std::move(deserializerFactory)); auto iter = reader->readStream(in); while (iter->hasNext()) { @@ -316,15 +378,12 @@ class SinglePartitioningShuffleWriter : public VeloxShuffleWriterTest { } std::shared_ptr createShuffleWriter(arrow::MemoryPool* arrowPool) override { - shuffleWriterOptions_.bufferSize = 10; shuffleWriterOptions_.partitioning = Partitioning::kSingle; static const uint32_t kNumPartitions = 1; auto partitionWriter = createPartitionWriter( GetParam().partitionWriterType, kNumPartitions, dataFile_, localDirs_, partitionWriterOptions_, arrowPool); - GLUTEN_ASSIGN_OR_THROW( - auto shuffleWriter, - VeloxShuffleWriter::create( - kNumPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions_), pool_, arrowPool)); + std::shared_ptr shuffleWriter = createSpecificShuffleWriter( + arrowPool, std::move(partitionWriter), std::move(shuffleWriterOptions_), kNumPartitions, 10); return shuffleWriter; } }; @@ -387,15 +446,12 @@ class HashPartitioningShuffleWriter : public MultiplePartitioningShuffleWriter { } std::shared_ptr createShuffleWriter(arrow::MemoryPool* arrowPool) override { - shuffleWriterOptions_.bufferSize = 4; shuffleWriterOptions_.partitioning = Partitioning::kHash; static const uint32_t kNumPartitions = 2; auto partitionWriter = createPartitionWriter( GetParam().partitionWriterType, kNumPartitions, dataFile_, localDirs_, partitionWriterOptions_, arrowPool); - GLUTEN_ASSIGN_OR_THROW( - auto shuffleWriter, - VeloxShuffleWriter::create( - kNumPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions_), pool_, arrowPool)); + std::shared_ptr shuffleWriter = createSpecificShuffleWriter( + arrowPool, std::move(partitionWriter), std::move(shuffleWriterOptions_), kNumPartitions, 4); return shuffleWriter; } @@ -422,15 +478,12 @@ class RangePartitioningShuffleWriter : public MultiplePartitioningShuffleWriter } std::shared_ptr createShuffleWriter(arrow::MemoryPool* arrowPool) override { - shuffleWriterOptions_.bufferSize = 4; shuffleWriterOptions_.partitioning = Partitioning::kRange; static const uint32_t kNumPartitions = 2; auto partitionWriter = createPartitionWriter( GetParam().partitionWriterType, kNumPartitions, dataFile_, localDirs_, partitionWriterOptions_, arrowPool); - GLUTEN_ASSIGN_OR_THROW( - auto shuffleWriter, - VeloxShuffleWriter::create( - kNumPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions_), pool_, arrowPool)); + std::shared_ptr shuffleWriter = createSpecificShuffleWriter( + arrowPool, std::move(partitionWriter), std::move(shuffleWriterOptions_), kNumPartitions, 4); return shuffleWriter; } @@ -441,7 +494,7 @@ class RangePartitioningShuffleWriter : public MultiplePartitioningShuffleWriter facebook::velox::TypePtr dataType, std::vector> expectedVectors) { /* blockId = pid, rowVector in block */ for (auto& batch : batches) { - ASSERT_NOT_OK(shuffleWriter.split(batch, ShuffleWriter::kMinMemLimit)); + ASSERT_NOT_OK(shuffleWriter.write(batch, ShuffleWriter::kMinMemLimit)); } shuffleWriteReadMultiBlocks(shuffleWriter, expectPartitionLength, dataType, expectedVectors); } @@ -453,14 +506,11 @@ class RangePartitioningShuffleWriter : public MultiplePartitioningShuffleWriter class RoundRobinPartitioningShuffleWriter : public MultiplePartitioningShuffleWriter { protected: std::shared_ptr createShuffleWriter(arrow::MemoryPool* arrowPool) override { - shuffleWriterOptions_.bufferSize = 4; static const uint32_t kNumPartitions = 2; auto partitionWriter = createPartitionWriter( GetParam().partitionWriterType, kNumPartitions, dataFile_, localDirs_, partitionWriterOptions_, arrowPool); - GLUTEN_ASSIGN_OR_THROW( - auto shuffleWriter, - VeloxShuffleWriter::create( - kNumPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions_), pool_, arrowPool)); + std::shared_ptr shuffleWriter = createSpecificShuffleWriter( + arrowPool, std::move(partitionWriter), std::move(shuffleWriterOptions_), kNumPartitions, 4); return shuffleWriter; } }; @@ -479,7 +529,7 @@ class VeloxShuffleWriterMemoryTest : public VeloxShuffleWriterTestBase, public t PartitionWriterType::kLocal, numPartitions, dataFile_, localDirs_, partitionWriterOptions_, arrowPool); GLUTEN_ASSIGN_OR_THROW( auto shuffleWriter, - VeloxShuffleWriter::create( + VeloxHashBasedShuffleWriter::create( numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions_), pool_, arrowPool)); return shuffleWriter; } diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index a1a41f973249..f454cf00c656 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -24,7 +24,6 @@ import org.apache.celeborn.client.LifecycleManager; import org.apache.celeborn.client.ShuffleClient; import org.apache.celeborn.common.CelebornConf; -import org.apache.celeborn.common.protocol.ShuffleMode; import org.apache.spark.*; import org.apache.spark.shuffle.*; import org.apache.spark.shuffle.celeborn.*; @@ -291,10 +290,6 @@ public ShuffleWriter getWriter( shuffleId = h.dependency().shuffleId(); } - if (!ShuffleMode.HASH.equals(celebornConf.shuffleWriterMode())) { - throw new UnsupportedOperationException( - "Unrecognized shuffle write mode!" + celebornConf.shuffleWriterMode()); - } if (h.dependency() instanceof ColumnarShuffleDependency) { // columnar-based shuffle return writerFactory.createShuffleWriterInstance( diff --git a/gluten-celeborn/common/src/main/scala/org/apache/spark/shuffle/CelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/common/src/main/scala/org/apache/spark/shuffle/CelebornHashBasedColumnarShuffleWriter.scala index 292ff3cc1fc8..efd891498131 100644 --- a/gluten-celeborn/common/src/main/scala/org/apache/spark/shuffle/CelebornHashBasedColumnarShuffleWriter.scala +++ b/gluten-celeborn/common/src/main/scala/org/apache/spark/shuffle/CelebornHashBasedColumnarShuffleWriter.scala @@ -29,6 +29,7 @@ import org.apache.celeborn.client.ShuffleClient import org.apache.celeborn.common.CelebornConf import java.io.IOException +import java.util.Locale abstract class CelebornHashBasedColumnarShuffleWriter[K, V]( shuffleId: Int, @@ -53,6 +54,13 @@ abstract class CelebornHashBasedColumnarShuffleWriter[K, V]( protected val clientPushBufferMaxSize: Int = celebornConf.clientPushBufferMaxSize + protected val clientPushSortMemoryThreshold: Long = celebornConf.clientPushSortMemoryThreshold + + protected val clientSortMemoryMaxSize: Long = celebornConf.clientPushSortMemoryThreshold + + protected val shuffleWriterType: String = + celebornConf.shuffleWriterMode.name.toLowerCase(Locale.ROOT) + protected val celebornPartitionPusher = new CelebornPartitionPusher( shuffleId, numMappers, diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index d72977f59714..699626db12c5 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -39,6 +39,7 @@ import org.apache.celeborn.client.read.CelebornInputStream import java.io._ import java.nio.ByteBuffer +import java.util.Locale import java.util.UUID import java.util.concurrent.atomic.AtomicBoolean @@ -83,6 +84,8 @@ private class CelebornColumnarBatchSerializerInstance( } val compressionCodecBackend = GlutenConfig.getConf.columnarShuffleCodecBackend.orNull + val shuffleWriterType = + conf.get("spark.celeborn.client.spark.shuffle.writer", "hash").toLowerCase(Locale.ROOT) val jniWrapper = ShuffleReaderJniWrapper.create() val batchSize = GlutenConfig.getConf.maxBatchSize val handle = jniWrapper @@ -91,7 +94,8 @@ private class CelebornColumnarBatchSerializerInstance( nmm.getNativeInstanceHandle, compressionCodec, compressionCodecBackend, - batchSize + batchSize, + shuffleWriterType ) // Close shuffle reader instance as lately as the end of task processing, // since the native reader could hold a reference to memory pool that diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala index 75d5148cd366..37ea11a73d2a 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala @@ -69,6 +69,12 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( } } + private val memoryLimit: Long = if ("sort".equals(shuffleWriterType)) { + Math.min(clientSortMemoryMaxSize, clientPushBufferMaxSize * numPartitions) + } else { + availableOffHeapPerTask() + } + private def availableOffHeapPerTask(): Long = { val perTask = SparkMemoryUtil.getCurrentAvailableOffHeapMemory / SparkResourceUtil.getTaskSlots(conf) @@ -97,6 +103,7 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( bufferCompressThreshold, GlutenConfig.getConf.columnarShuffleCompressionMode, clientPushBufferMaxSize, + clientPushSortMemoryThreshold, celebornPartitionPusher, NativeMemoryManagers .create( @@ -127,11 +134,12 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( context.taskAttemptId(), GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, context.partitionId), "celeborn", + shuffleWriterType, GlutenConfig.getConf.columnarShuffleReallocThreshold ) } val startTime = System.nanoTime() - jniWrapper.split(nativeShuffleWriter, cb.numRows, handle, availableOffHeapPerTask()) + jniWrapper.write(nativeShuffleWriter, cb.numRows, handle, availableOffHeapPerTask()) dep.metrics("splitTime").add(System.nanoTime() - startTime) dep.metrics("numInputRows").add(cb.numRows) dep.metrics("inputBatches").add(1) diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java index 411907ae3430..24425ccf72e6 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java @@ -41,7 +41,8 @@ public native long make( long memoryManagerHandle, String compressionType, String compressionCodecBackend, - int batchSize); + int batchSize, + String shuffleWriterType); public native long readStream(long shuffleReaderHandle, JniByteInputStream jniIn); diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java index 243c90599f5e..ed312fa14b24 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java @@ -90,8 +90,10 @@ public long make( taskAttemptId, startPartitionId, 0, + 0, null, - "local"); + "local", + "hash"); } /** @@ -110,12 +112,14 @@ public long makeForRSS( int bufferCompressThreshold, String compressionMode, int pushBufferMaxSize, + long sortBufferMaxSize, Object pusher, long memoryManagerHandle, long handle, long taskAttemptId, int startPartitionId, String partitionWriterType, + String shuffleWriterType, double reallocThreshold) { return nativeMake( part.getShortName(), @@ -137,8 +141,10 @@ public long makeForRSS( taskAttemptId, startPartitionId, pushBufferMaxSize, + sortBufferMaxSize, pusher, - partitionWriterType); + partitionWriterType, + shuffleWriterType); } public native long nativeMake( @@ -161,8 +167,10 @@ public native long nativeMake( long taskAttemptId, int startPartitionId, int pushBufferMaxSize, + long sortBufferMaxSize, Object pusher, - String partitionWriterType); + String partitionWriterType, + String shuffleWriterType); /** * Evict partition data. @@ -187,7 +195,7 @@ public native long nativeEvict(long shuffleWriterHandle, long size, boolean call * allocator instead * @return batch bytes. */ - public native long split(long shuffleWriterHandle, int numRows, long handler, long memLimit); + public native long write(long shuffleWriterHandle, int numRows, long handler, long memLimit); /** * Write the data remained in the buffers hold by native shuffle writer to each partition's diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index e632700e3743..69e9aa9c951a 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -102,7 +102,9 @@ private class ColumnarBatchSerializerInstance( nmm.getNativeInstanceHandle, compressionCodec, compressionCodecBackend, - batchSize) + batchSize, + "hash" + ) // Close shuffle reader instance as lately as the end of task processing, // since the native reader could hold a reference to memory pool that // was used to create all buffers read from shuffle reader. The pool diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala index fb933866cea7..c797257f1fd1 100644 --- a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala +++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala @@ -181,7 +181,7 @@ class ColumnarShuffleWriter[K, V]( ) } val startTime = System.nanoTime() - jniWrapper.split(nativeShuffleWriter, rows, handle, availableOffHeapPerTask()) + jniWrapper.write(nativeShuffleWriter, rows, handle, availableOffHeapPerTask()) dep.metrics("splitTime").add(System.nanoTime() - startTime) dep.metrics("numInputRows").add(rows) dep.metrics("inputBatches").add(1) diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index 17cfce1c0c3e..c0063c6f4274 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -145,6 +145,7 @@ protected void writeImpl(Iterator> records) throws IOException { compressThreshold, GlutenConfig.getConf().columnarShuffleCompressionMode(), bufferSize, + bufferSize, partitionPusher, NativeMemoryManagers.create( "UniffleShuffleWriter", @@ -180,12 +181,13 @@ public Set applicablePhases() { GlutenShuffleUtils.getStartPartitionId( columnarDep.nativePartitioning(), partitionId), "uniffle", + "hash", reallocThreshold); } long startTime = System.nanoTime(); long bytes = - jniWrapper.split(nativeShuffleWriter, cb.numRows(), handle, availableOffHeapPerTask()); - LOG.debug("jniWrapper.split rows {}, split bytes {}", cb.numRows(), bytes); + jniWrapper.write(nativeShuffleWriter, cb.numRows(), handle, availableOffHeapPerTask()); + LOG.debug("jniWrapper.write rows {}, split bytes {}", cb.numRows(), bytes); columnarDep.metrics().get("dataSize").get().add(bytes); // this metric replace part of uniffle shuffle write time columnarDep.metrics().get("splitTime").get().add(System.nanoTime() - startTime); diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index ca8a9dce12c5..02c6bf7fe4ac 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -134,6 +134,11 @@ class GlutenConfig(conf: SQLConf) extends Logging { .getConfString("spark.shuffle.manager", "sort") .contains("UniffleShuffleManager") + def isSortBasedCelebornShuffle: Boolean = + conf + .getConfString("spark.celeborn.client.spark.shuffle.writer", "hash") + .equals("sort") + def enableColumnarShuffle: Boolean = conf.getConf(COLUMNAR_SHUFFLE_ENABLED) def enablePreferColumnar: Boolean = conf.getConf(COLUMNAR_PREFER_ENABLED) From a4b69a2f141bbc9eede669cc337112845cd712ef Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Fri, 17 May 2024 14:06:03 +0800 Subject: [PATCH 093/402] [CORE] Add decimal precision tests (#5752) * Add decimal precision tests * fix ck test * fix * fix --------- Co-authored-by: Kent Yao --- .../expression/CHExpressionTransformer.scala | 10 +- .../expression/ExpressionTransformer.scala | 10 +- .../spark/sql/expression/UDFResolver.scala | 32 ++-- .../gluten/backendsapi/SparkPlanExecApi.scala | 8 - .../ArrayExpressionTransformer.scala | 4 +- .../expression/ConditionalTransformer.scala | 4 +- .../DateTimeExpressionsTransformer.scala | 10 +- .../expression/ExpressionConverter.scala | 7 - .../expression/ExpressionTransformer.scala | 9 ++ .../GenericExpressionTransformer.scala | 2 +- .../HashExpressionTransformer.scala | 2 +- .../JsonTupleExpressionTransformer.scala | 2 +- .../LambdaFunctionTransformer.scala | 2 +- .../expression/LiteralTransformer.scala | 4 +- .../expression/MapExpressionTransformer.scala | 4 +- .../NamedExpressionsTransformer.scala | 2 +- .../PredicateExpressionTransformer.scala | 9 +- .../ScalarSubqueryTransformer.scala | 3 +- .../StringExpressionTransformer.scala | 2 +- .../StructExpressionTransformer.scala | 2 +- .../UnaryExpressionTransformer.scala | 17 ++- .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 1 + .../GlutenDecimalPrecisionSuite.scala | 138 ++++++++++++++++++ .../CustomerExpressionTransformer.scala | 4 +- .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 3 +- .../GlutenDecimalPrecisionSuite.scala | 138 ++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 3 +- .../GlutenDecimalPrecisionSuite.scala | 138 ++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 3 +- .../GlutenDecimalPrecisionSuite.scala | 138 ++++++++++++++++++ 34 files changed, 639 insertions(+), 76 deletions(-) create mode 100644 gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala create mode 100644 gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala index 7d9dbaddc073..98cc4a930d2f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala @@ -34,7 +34,7 @@ case class CHSizeExpressionTransformer( substraitExprName: String, child: ExpressionTransformer, original: Size) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // Pass legacyLiteral as second argument in substrait function @@ -51,7 +51,7 @@ case class CHTruncTimestampTransformer( timestamp: ExpressionTransformer, timeZoneId: Option[String] = None, original: TruncTimestamp) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // The format must be constant string in the function date_trunc of ch. @@ -126,7 +126,7 @@ case class CHStringTranslateTransformer( matchingExpr: ExpressionTransformer, replaceExpr: ExpressionTransformer, original: StringTranslate) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // In CH, translateUTF8 requires matchingExpr and replaceExpr argument have the same length @@ -158,7 +158,7 @@ case class CHPosExplodeTransformer( child: ExpressionTransformer, original: PosExplode, attributeSeq: Seq[Attribute]) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode: ExpressionNode = child.doTransform(args) @@ -202,7 +202,7 @@ case class CHRegExpReplaceTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], original: RegExpReplace) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // In CH: replaceRegexpAll(subject, regexp, rep), which is equivalent diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala index 75a2c3a62da5..da8433fa2e48 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala @@ -35,7 +35,7 @@ case class VeloxAliasTransformer( substraitExprName: String, child: ExpressionTransformer, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { child.doTransform(args) @@ -46,7 +46,7 @@ case class VeloxNamedStructTransformer( substraitExprName: String, original: CreateNamedStruct, attributeSeq: Seq[Attribute]) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: Object): ExpressionNode = { val expressionNodes = Lists.newArrayList[ExpressionNode]() original.valExprs.foreach( @@ -67,7 +67,7 @@ case class VeloxGetStructFieldTransformer( childTransformer: ExpressionTransformer, ordinal: Int, original: GetStructField) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: Object): ExpressionNode = { val childNode = childTransformer.doTransform(args) childNode match { @@ -86,7 +86,7 @@ case class VeloxHashExpressionTransformer( substraitExprName: String, exps: Seq[ExpressionTransformer], original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // As of Spark 3.3, there are 3 kinds of HashExpression. // HiveHash is not supported in native backend and will fail native validation. @@ -121,7 +121,7 @@ case class VeloxStringSplitTransformer( regexExpr: ExpressionTransformer, limitExpr: ExpressionTransformer, original: StringSplit) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { if ( diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index bdfd24ed5c1b..847e5a2e683e 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.velox.VeloxBackendSettings import org.apache.gluten.exception.GlutenException import org.apache.gluten.expression.{ConverterUtils, ExpressionTransformer, ExpressionType, Transformable} import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.ExpressionBuilder +import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.gluten.udf.UdfJniWrapper import org.apache.gluten.vectorized.JniWorkspace @@ -110,18 +110,24 @@ case class UDFExpression( this.getClass.getSimpleName + ": getTransformer called before children transformer initialized.") } - (args: Object) => { - val transformers = childrenTransformers.map(_.doTransform(args)) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(name, children.map(_.dataType), FunctionConfig.REQ)) - - val typeNode = ConverterUtils.getTypeNode(dataType, nullable) - ExpressionBuilder.makeScalarFunction( - functionId, - Lists.newArrayList(transformers: _*), - typeNode) + + val localDataType = dataType + new ExpressionTransformer { + override def doTransform(args: Object): ExpressionNode = { + val transformers = childrenTransformers.map(_.doTransform(args)) + val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] + val functionId = ExpressionBuilder.newScalarFunction( + functionMap, + ConverterUtils.makeFuncName(name, children.map(_.dataType), FunctionConfig.REQ)) + + val typeNode = ConverterUtils.getTypeNode(dataType, nullable) + ExpressionBuilder.makeScalarFunction( + functionId, + Lists.newArrayList(transformers: _*), + typeNode) + } + + override def dataType: DataType = localDataType } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 8df74bb88bfe..aa27d1ce1865 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -451,14 +451,6 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, children, original) } - def genEqualNullSafeTransformer( - substraitExprName: String, - left: ExpressionTransformer, - right: ExpressionTransformer, - original: EqualNullSafe): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(left, right), original) - } - def genMd5Transformer( substraitExprName: String, child: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala index 85a1f58fba20..68a464f13222 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala @@ -33,7 +33,7 @@ case class CreateArrayTransformer( children: Seq[ExpressionTransformer], useStringTypeWhenEmpty: Boolean, original: CreateArray) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // If children is empty, @@ -62,7 +62,7 @@ case class GetArrayItemTransformer( right: ExpressionTransformer, failOnError: Boolean, original: GetArrayItem) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // Ignore failOnError for clickhouse backend diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala index 18a46d7cad63..0fdd68511eec 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala @@ -27,7 +27,7 @@ case class CaseWhenTransformer( branches: Seq[(ExpressionTransformer, ExpressionTransformer)], elseValue: Option[ExpressionTransformer], original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // generate branches nodes @@ -52,7 +52,7 @@ case class IfTransformer( trueValue: ExpressionTransformer, falseValue: ExpressionTransformer, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val ifNodes = new JArrayList[ExpressionNode] diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala index 797dc81d3f69..66004291ac4e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala @@ -36,7 +36,7 @@ case class ExtractDateTransformer( substraitExprName: String, child: ExpressionTransformer, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode = child.doTransform(args) @@ -65,7 +65,7 @@ case class DateDiffTransformer( endDate: ExpressionTransformer, startDate: ExpressionTransformer, original: DateDiff) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val endDateNode = endDate.doTransform(args) @@ -99,7 +99,7 @@ case class ToUnixTimestampTransformer( timeZoneId: Option[String], failOnError: Boolean, original: ToUnixTimestamp) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val dataTypes = Seq(original.timeExp.dataType, StringType) @@ -124,7 +124,7 @@ case class TruncTimestampTransformer( timestamp: ExpressionTransformer, timeZoneId: Option[String] = None, original: TruncTimestamp) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val timestampNode = timestamp.doTransform(args) @@ -160,7 +160,7 @@ case class MonthsBetweenTransformer( roundOff: ExpressionTransformer, timeZoneId: Option[String] = None, original: MonthsBetween) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val date1Node = date1.doTransform(args) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index b7b946268ff5..e692890c452b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -415,13 +415,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { ), r ) - case equal: EqualNullSafe => - BackendsApiManager.getSparkPlanExecApiInstance.genEqualNullSafeTransformer( - substraitExprName, - replaceWithExpressionTransformerInternal(equal.left, attributeSeq, expressionsMap), - replaceWithExpressionTransformerInternal(equal.right, attributeSeq, expressionsMap), - equal - ) case md5: Md5 => BackendsApiManager.getSparkPlanExecApiInstance.genMd5Transformer( substraitExprName, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala index 65badcbaea44..6b65878627c5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala @@ -18,6 +18,15 @@ package org.apache.gluten.expression import org.apache.gluten.substrait.expression.ExpressionNode +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.types.DataType + trait ExpressionTransformer { def doTransform(args: java.lang.Object): ExpressionNode + def dataType: DataType +} + +trait ExpressionTransformerWithOrigin extends ExpressionTransformer { + def original: Expression + def dataType: DataType = original.dataType } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala index 62afcad2838a..8faf4965fb4d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala @@ -27,7 +27,7 @@ case class GenericExpressionTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: Object): ExpressionNode = { val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] val functionId = ExpressionBuilder.newScalarFunction( diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala index d813f8250696..28f2dda01e61 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala @@ -25,7 +25,7 @@ case class HashExpressionTransformer( substraitExprName: String, exps: Seq[ExpressionTransformer], original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val nodes = new java.util.ArrayList[ExpressionNode]() diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala index 25e3e12a53de..e8ff3d360a8d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala @@ -28,7 +28,7 @@ case class JsonTupleExpressionTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: Object): ExpressionNode = { val jsonExpr = children.head diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala index 492de2b76a4a..ce6d13a95181 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala @@ -27,7 +27,7 @@ case class LambdaFunctionTransformer( arguments: Seq[ExpressionTransformer], hidden: Boolean = false, original: LambdaFunction) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: Object): ExpressionNode = { // Need to fallback when hidden be true as it's not supported in Velox diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala index 05787858e825..8fb9943d6398 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala @@ -20,9 +20,9 @@ import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode import org.apache.spark.sql.catalyst.expressions._ -case class LiteralTransformer(lit: Literal) extends ExpressionTransformer { +case class LiteralTransformer(original: Literal) extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { - ExpressionBuilder.makeLiteral(lit.value, lit.dataType, lit.nullable) + ExpressionBuilder.makeLiteral(original.value, original.dataType, original.nullable) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala index e136f1b3a31a..c09afaebc35a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala @@ -30,7 +30,7 @@ case class CreateMapTransformer( children: Seq[ExpressionTransformer], useStringTypeWhenEmpty: Boolean, original: CreateMap) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { // If children is empty, @@ -64,7 +64,7 @@ case class GetMapValueTransformer( key: ExpressionTransformer, failOnError: Boolean, original: GetMapValue) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { if (BackendsApiManager.getSettings.alwaysFailOnMapExpression()) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala index 70ad13584edd..2af4a5fa2558 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala @@ -28,7 +28,7 @@ case class AliasTransformer( substraitExprName: String, child: ExpressionTransformer, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode = child.doTransform(args) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala index dfa4ceed6055..7d34466e5044 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala @@ -32,7 +32,7 @@ case class InTransformer( list: Seq[Expression], valueType: DataType, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { assert(list.forall(_.foldable)) // Stores the values in a List Literal. @@ -46,7 +46,7 @@ case class InSetTransformer( hset: Set[Any], valueType: DataType, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { InExpressionTransformer.toTransformer(value.doTransform(args), hset, valueType) } @@ -74,7 +74,7 @@ case class LikeTransformer( left: ExpressionTransformer, right: ExpressionTransformer, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val leftNode = left.doTransform(args) val rightNode = right.doTransform(args) @@ -108,7 +108,8 @@ case class DecimalArithmeticExpressionTransformer( right: ExpressionTransformer, resultType: DecimalType, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { + override def dataType: DataType = resultType override def doTransform(args: java.lang.Object): ExpressionNode = { val leftNode = left.doTransform(args) val rightNode = right.doTransform(args) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala index 534bde3b3a48..4f5a43d47646 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala @@ -22,7 +22,8 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.{BaseSubqueryExec, ScalarSubquery} case class ScalarSubqueryTransformer(plan: BaseSubqueryExec, exprId: ExprId, query: ScalarSubquery) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { + override def original: Expression = query override def doTransform(args: java.lang.Object): ExpressionNode = { // don't trigger collect when in validation phase diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala index da021be24834..b31d66b68e0a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala @@ -28,7 +28,7 @@ case class String2TrimExpressionTransformer( trimStr: Option[ExpressionTransformer], srcStr: ExpressionTransformer, original: Expression) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val trimStrNode = trimStr.map(_.doTransform(args)) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala index c70395a7d7a7..616971b6d15f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala @@ -29,7 +29,7 @@ case class GetStructFieldTransformer( childTransformer: ExpressionTransformer, ordinal: Int, original: GetStructField) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode = childTransformer.doTransform(args) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala index 2d3840ce4f03..d0ac19b4a9e3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala @@ -32,17 +32,18 @@ case class ChildTransformer(child: ExpressionTransformer) extends ExpressionTran override def doTransform(args: java.lang.Object): ExpressionNode = { child.doTransform(args) } + override def dataType: DataType = child.dataType } case class CastTransformer( child: ExpressionTransformer, - datatype: DataType, + dataType: DataType, timeZoneId: Option[String], original: Cast) extends ExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { - val typeNode = ConverterUtils.getTypeNode(datatype, original.nullable) + val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable) ExpressionBuilder.makeCast(typeNode, child.doTransform(args), original.ansiEnabled) } } @@ -51,7 +52,7 @@ case class ExplodeTransformer( substraitExprName: String, child: ExpressionTransformer, original: Explode) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode: ExpressionNode = child.doTransform(args) @@ -79,7 +80,7 @@ case class PosExplodeTransformer( child: ExpressionTransformer, original: PosExplode, attributeSeq: Seq[Attribute]) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode: ExpressionNode = child.doTransform(args) @@ -154,7 +155,7 @@ case class CheckOverflowTransformer( child: ExpressionTransformer, childResultType: DataType, original: CheckOverflow) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { BackendsApiManager.getTransformerApiInstance.createCheckOverflowExprNode( @@ -172,7 +173,7 @@ case class MakeDecimalTransformer( substraitExprName: String, child: ExpressionTransformer, original: MakeDecimal) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode = child.doTransform(args) @@ -202,7 +203,7 @@ case class RandTransformer( substraitExprName: String, explicitSeed: ExpressionTransformer, original: Rand) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { if (!original.hideSeed) { @@ -226,7 +227,7 @@ case class GetArrayStructFieldsTransformer( numFields: Int, containsNull: Boolean, original: GetArrayStructFields) - extends ExpressionTransformer { + extends ExpressionTransformerWithOrigin { override def doTransform(args: java.lang.Object): ExpressionNode = { val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index bc0410834dd9..afc427cd3664 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -756,6 +756,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeGlutenTest("to_unix_timestamp") .excludeGlutenTest("Hour") enableSuite[GlutenDecimalExpressionSuite] + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] .exclude("sha2") .exclude("murmur3/xxHash64/hive hash: struct") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 366796a57465..5e35912034b8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -226,6 +226,7 @@ class VeloxTestSettings extends BackendTestSettings { // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") enableSuite[GlutenDecimalExpressionSuite] + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenStringFunctionsSuite] enableSuite[GlutenRegexpExpressionsSuite] enableSuite[GlutenNullExpressionsSuite] diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala new file mode 100644 index 000000000000..97e752d7d046 --- /dev/null +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.gluten.expression._ + +import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} +import org.apache.spark.sql.types._ + +class GlutenDecimalPrecisionSuite extends GlutenTestsTrait { + private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry) + private val analyzer = new Analyzer(catalog) + + private val relation = LocalRelation( + AttributeReference("i", IntegerType)(), + AttributeReference("d1", DecimalType(2, 1))(), + AttributeReference("d2", DecimalType(5, 2))(), + AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), + AttributeReference("f", FloatType)(), + AttributeReference("b", DoubleType)() + ) + + private val i: Expression = UnresolvedAttribute("i") + private val d1: Expression = UnresolvedAttribute("d1") + private val d2: Expression = UnresolvedAttribute("d2") + private val u: Expression = UnresolvedAttribute("u") + private val f: Expression = UnresolvedAttribute("f") + private val b: Expression = UnresolvedAttribute("b") + + private def checkType(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Seq(Alias(expression, "c")()), relation)) + assert(plan.isInstanceOf[Project]) + val expr = plan.asInstanceOf[Project].projectList.head + assert(expr.dataType == expectedType) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.dataType == expectedType) + } + + private def stripAlias(expr: Expression): Expression = { + expr match { + case a: Alias => stripAlias(a.child) + case _ => expr + } + } + + private def checkComparison(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Alias(expression, "c")() :: Nil, relation)) + assert(plan.isInstanceOf[Project]) + val expr = stripAlias(plan.asInstanceOf[Project].projectList.head) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.isInstanceOf[GenericExpressionTransformer]) + val binaryComparison = transformedExpr.asInstanceOf[GenericExpressionTransformer] + assert(binaryComparison.original.isInstanceOf[BinaryComparison]) + assert(binaryComparison.children.size == 2) + assert(binaryComparison.children.forall(_.dataType == expectedType)) + } + + test("basic operations") { + checkType(Add(d1, d2), DecimalType(6, 2)) + checkType(Subtract(d1, d2), DecimalType(6, 2)) + checkType(Multiply(d1, d2), DecimalType(8, 3)) + checkType(Divide(d1, d2), DecimalType(10, 7)) + checkType(Divide(d2, d1), DecimalType(10, 6)) + + checkType(Add(Add(d1, d2), d1), DecimalType(7, 2)) + checkType(Add(Add(d1, d1), d1), DecimalType(4, 1)) + checkType(Add(d1, Add(d1, d1)), DecimalType(4, 1)) + checkType(Add(Add(Add(d1, d2), d1), d2), DecimalType(8, 2)) + checkType(Add(Add(d1, d2), Add(d1, d2)), DecimalType(7, 2)) + checkType(Subtract(Subtract(d2, d1), d1), DecimalType(7, 2)) + checkType(Multiply(Multiply(d1, d1), d2), DecimalType(11, 4)) + checkType(Divide(d2, Add(d1, d1)), DecimalType(10, 6)) + } + + test("Comparison operations") { + checkComparison(EqualTo(i, d1), DecimalType(11, 1)) + checkComparison(EqualNullSafe(d2, d1), DecimalType(5, 2)) + checkComparison(LessThan(i, d1), DecimalType(11, 1)) + checkComparison(LessThanOrEqual(d1, d2), DecimalType(5, 2)) + checkComparison(GreaterThan(d2, u), DecimalType.SYSTEM_DEFAULT) + checkComparison(GreaterThanOrEqual(d1, f), DoubleType) + checkComparison(GreaterThan(d2, d2), DecimalType(5, 2)) + } + + test("bringing in primitive types") { + checkType(Add(d1, i), DecimalType(12, 1)) + checkType(Add(d1, f), DoubleType) + checkType(Add(i, d1), DecimalType(12, 1)) + checkType(Add(f, d1), DoubleType) + checkType(Add(d1, Cast(i, LongType)), DecimalType(22, 1)) + checkType(Add(d1, Cast(i, ShortType)), DecimalType(7, 1)) + checkType(Add(d1, Cast(i, ByteType)), DecimalType(5, 1)) + checkType(Add(d1, Cast(i, DoubleType)), DoubleType) + } + + test("maximum decimals") { + for (expr <- Seq(d1, d2, i, u)) { + checkType(Add(expr, u), DecimalType(38, 17)) + checkType(Subtract(expr, u), DecimalType(38, 17)) + } + + checkType(Multiply(d1, u), DecimalType(38, 16)) + checkType(Multiply(d2, u), DecimalType(38, 14)) + checkType(Multiply(i, u), DecimalType(38, 7)) + checkType(Multiply(u, u), DecimalType(38, 6)) + + checkType(Divide(u, d1), DecimalType(38, 17)) + checkType(Divide(u, d2), DecimalType(38, 16)) + checkType(Divide(u, i), DecimalType(38, 18)) + checkType(Divide(u, u), DecimalType(38, 6)) + + for (expr <- Seq(f, b)) { + checkType(Add(expr, u), DoubleType) + checkType(Subtract(expr, u), DoubleType) + checkType(Multiply(expr, u), DoubleType) + checkType(Divide(expr, u), DoubleType) + } + } +} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala index a3720fc6235d..c27159cebdda 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala @@ -26,12 +26,12 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import com.google.common.collect.Lists -class CustomAddExpressionTransformer( +case class CustomAddExpressionTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, original: Expression) - extends ExpressionTransformer + extends ExpressionTransformerWithOrigin with Logging { override def doTransform(args: java.lang.Object): ExpressionNode = { val leftNode = left.doTransform(args) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 6a403204fb7a..85f3f94cca95 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -800,6 +800,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeGlutenTest("to_unix_timestamp") .excludeGlutenTest("Hour") enableSuite[GlutenDecimalExpressionSuite] + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] .exclude("sha2") .exclude("murmur3/xxHash64/hive hash: struct") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 128e52a79b77..1d796aa1b74a 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -19,7 +19,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.{GlutenAnsiCastSuiteWithAnsiModeOff, GlutenAnsiCastSuiteWithAnsiModeOn, GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCastSuiteWithAnsiModeOn, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite, GlutenTryCastSuite} +import org.apache.spark.sql.catalyst.expressions.{GlutenAnsiCastSuiteWithAnsiModeOff, GlutenAnsiCastSuiteWithAnsiModeOn, GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCastSuiteWithAnsiModeOn, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite, GlutenTryCastSuite} import org.apache.spark.sql.connector._ import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} import org.apache.spark.sql.execution._ @@ -141,6 +141,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("from_unixtime") .exclude("test timestamp add") enableSuite[GlutenDecimalExpressionSuite] + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] enableSuite[GlutenHigherOrderFunctionsSuite] enableSuite[GlutenIntervalExpressionsSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala new file mode 100644 index 000000000000..97e752d7d046 --- /dev/null +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.gluten.expression._ + +import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} +import org.apache.spark.sql.types._ + +class GlutenDecimalPrecisionSuite extends GlutenTestsTrait { + private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry) + private val analyzer = new Analyzer(catalog) + + private val relation = LocalRelation( + AttributeReference("i", IntegerType)(), + AttributeReference("d1", DecimalType(2, 1))(), + AttributeReference("d2", DecimalType(5, 2))(), + AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), + AttributeReference("f", FloatType)(), + AttributeReference("b", DoubleType)() + ) + + private val i: Expression = UnresolvedAttribute("i") + private val d1: Expression = UnresolvedAttribute("d1") + private val d2: Expression = UnresolvedAttribute("d2") + private val u: Expression = UnresolvedAttribute("u") + private val f: Expression = UnresolvedAttribute("f") + private val b: Expression = UnresolvedAttribute("b") + + private def checkType(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Seq(Alias(expression, "c")()), relation)) + assert(plan.isInstanceOf[Project]) + val expr = plan.asInstanceOf[Project].projectList.head + assert(expr.dataType == expectedType) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.dataType == expectedType) + } + + private def stripAlias(expr: Expression): Expression = { + expr match { + case a: Alias => stripAlias(a.child) + case _ => expr + } + } + + private def checkComparison(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Alias(expression, "c")() :: Nil, relation)) + assert(plan.isInstanceOf[Project]) + val expr = stripAlias(plan.asInstanceOf[Project].projectList.head) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.isInstanceOf[GenericExpressionTransformer]) + val binaryComparison = transformedExpr.asInstanceOf[GenericExpressionTransformer] + assert(binaryComparison.original.isInstanceOf[BinaryComparison]) + assert(binaryComparison.children.size == 2) + assert(binaryComparison.children.forall(_.dataType == expectedType)) + } + + test("basic operations") { + checkType(Add(d1, d2), DecimalType(6, 2)) + checkType(Subtract(d1, d2), DecimalType(6, 2)) + checkType(Multiply(d1, d2), DecimalType(8, 3)) + checkType(Divide(d1, d2), DecimalType(10, 7)) + checkType(Divide(d2, d1), DecimalType(10, 6)) + + checkType(Add(Add(d1, d2), d1), DecimalType(7, 2)) + checkType(Add(Add(d1, d1), d1), DecimalType(4, 1)) + checkType(Add(d1, Add(d1, d1)), DecimalType(4, 1)) + checkType(Add(Add(Add(d1, d2), d1), d2), DecimalType(8, 2)) + checkType(Add(Add(d1, d2), Add(d1, d2)), DecimalType(7, 2)) + checkType(Subtract(Subtract(d2, d1), d1), DecimalType(7, 2)) + checkType(Multiply(Multiply(d1, d1), d2), DecimalType(11, 4)) + checkType(Divide(d2, Add(d1, d1)), DecimalType(10, 6)) + } + + test("Comparison operations") { + checkComparison(EqualTo(i, d1), DecimalType(11, 1)) + checkComparison(EqualNullSafe(d2, d1), DecimalType(5, 2)) + checkComparison(LessThan(i, d1), DecimalType(11, 1)) + checkComparison(LessThanOrEqual(d1, d2), DecimalType(5, 2)) + checkComparison(GreaterThan(d2, u), DecimalType.SYSTEM_DEFAULT) + checkComparison(GreaterThanOrEqual(d1, f), DoubleType) + checkComparison(GreaterThan(d2, d2), DecimalType(5, 2)) + } + + test("bringing in primitive types") { + checkType(Add(d1, i), DecimalType(12, 1)) + checkType(Add(d1, f), DoubleType) + checkType(Add(i, d1), DecimalType(12, 1)) + checkType(Add(f, d1), DoubleType) + checkType(Add(d1, Cast(i, LongType)), DecimalType(22, 1)) + checkType(Add(d1, Cast(i, ShortType)), DecimalType(7, 1)) + checkType(Add(d1, Cast(i, ByteType)), DecimalType(5, 1)) + checkType(Add(d1, Cast(i, DoubleType)), DoubleType) + } + + test("maximum decimals") { + for (expr <- Seq(d1, d2, i, u)) { + checkType(Add(expr, u), DecimalType(38, 17)) + checkType(Subtract(expr, u), DecimalType(38, 17)) + } + + checkType(Multiply(d1, u), DecimalType(38, 16)) + checkType(Multiply(d2, u), DecimalType(38, 14)) + checkType(Multiply(i, u), DecimalType(38, 7)) + checkType(Multiply(u, u), DecimalType(38, 6)) + + checkType(Divide(u, d1), DecimalType(38, 17)) + checkType(Divide(u, d2), DecimalType(38, 16)) + checkType(Divide(u, i), DecimalType(38, 18)) + checkType(Divide(u, u), DecimalType(38, 6)) + + for (expr <- Seq(f, b)) { + checkType(Add(expr, u), DoubleType) + checkType(Subtract(expr, u), DoubleType) + checkType(Multiply(expr, u), DoubleType) + checkType(Divide(expr, u), DoubleType) + } + } +} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 37e4c68f70f5..069d697bd454 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -639,6 +639,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeGlutenTest("to_unix_timestamp") .excludeGlutenTest("Hour") enableSuite[GlutenDecimalExpressionSuite].exclude("MakeDecimal") + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] .exclude("sha2") .exclude("murmur3/xxHash64/hive hash: struct") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 6ea29847b0a6..7c8509f8034d 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -19,7 +19,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} +import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} import org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite, GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite, GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter, GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite, GlutenDeleteFromTableSuite, GlutenDeltaBasedDeleteFromTableSuite, GlutenFileDataSourceV2FallBackSuite, GlutenGroupBasedDeleteFromTableSuite, GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, GlutenMetadataColumnSuite, GlutenSupportsCatalogOptionsSuite, GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite} import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} import org.apache.spark.sql.execution.{FallbackStrategiesSuite, GlutenBroadcastExchangeSuite, GlutenCoalesceShufflePartitionsSuite, GlutenExchangeSuite, GlutenLocalBroadcastExchangeSuite, GlutenReplaceHashWithSortAggSuite, GlutenReuseExchangeAndSubquerySuite, GlutenSameResultSuite, GlutenSortSuite, GlutenSQLAggregateFunctionSuite, GlutenSQLWindowFunctionSuite, GlutenTakeOrderedAndProjectSuite} @@ -121,6 +121,7 @@ class VeloxTestSettings extends BackendTestSettings { // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") enableSuite[GlutenDecimalExpressionSuite] + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] enableSuite[GlutenHigherOrderFunctionsSuite] enableSuite[GlutenIntervalExpressionsSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala new file mode 100644 index 000000000000..97e752d7d046 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.gluten.expression._ + +import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} +import org.apache.spark.sql.types._ + +class GlutenDecimalPrecisionSuite extends GlutenTestsTrait { + private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry) + private val analyzer = new Analyzer(catalog) + + private val relation = LocalRelation( + AttributeReference("i", IntegerType)(), + AttributeReference("d1", DecimalType(2, 1))(), + AttributeReference("d2", DecimalType(5, 2))(), + AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), + AttributeReference("f", FloatType)(), + AttributeReference("b", DoubleType)() + ) + + private val i: Expression = UnresolvedAttribute("i") + private val d1: Expression = UnresolvedAttribute("d1") + private val d2: Expression = UnresolvedAttribute("d2") + private val u: Expression = UnresolvedAttribute("u") + private val f: Expression = UnresolvedAttribute("f") + private val b: Expression = UnresolvedAttribute("b") + + private def checkType(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Seq(Alias(expression, "c")()), relation)) + assert(plan.isInstanceOf[Project]) + val expr = plan.asInstanceOf[Project].projectList.head + assert(expr.dataType == expectedType) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.dataType == expectedType) + } + + private def stripAlias(expr: Expression): Expression = { + expr match { + case a: Alias => stripAlias(a.child) + case _ => expr + } + } + + private def checkComparison(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Alias(expression, "c")() :: Nil, relation)) + assert(plan.isInstanceOf[Project]) + val expr = stripAlias(plan.asInstanceOf[Project].projectList.head) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.isInstanceOf[GenericExpressionTransformer]) + val binaryComparison = transformedExpr.asInstanceOf[GenericExpressionTransformer] + assert(binaryComparison.original.isInstanceOf[BinaryComparison]) + assert(binaryComparison.children.size == 2) + assert(binaryComparison.children.forall(_.dataType == expectedType)) + } + + test("basic operations") { + checkType(Add(d1, d2), DecimalType(6, 2)) + checkType(Subtract(d1, d2), DecimalType(6, 2)) + checkType(Multiply(d1, d2), DecimalType(8, 3)) + checkType(Divide(d1, d2), DecimalType(10, 7)) + checkType(Divide(d2, d1), DecimalType(10, 6)) + + checkType(Add(Add(d1, d2), d1), DecimalType(7, 2)) + checkType(Add(Add(d1, d1), d1), DecimalType(4, 1)) + checkType(Add(d1, Add(d1, d1)), DecimalType(4, 1)) + checkType(Add(Add(Add(d1, d2), d1), d2), DecimalType(8, 2)) + checkType(Add(Add(d1, d2), Add(d1, d2)), DecimalType(7, 2)) + checkType(Subtract(Subtract(d2, d1), d1), DecimalType(7, 2)) + checkType(Multiply(Multiply(d1, d1), d2), DecimalType(11, 4)) + checkType(Divide(d2, Add(d1, d1)), DecimalType(10, 6)) + } + + test("Comparison operations") { + checkComparison(EqualTo(i, d1), DecimalType(11, 1)) + checkComparison(EqualNullSafe(d2, d1), DecimalType(5, 2)) + checkComparison(LessThan(i, d1), DecimalType(11, 1)) + checkComparison(LessThanOrEqual(d1, d2), DecimalType(5, 2)) + checkComparison(GreaterThan(d2, u), DecimalType.SYSTEM_DEFAULT) + checkComparison(GreaterThanOrEqual(d1, f), DoubleType) + checkComparison(GreaterThan(d2, d2), DecimalType(5, 2)) + } + + test("bringing in primitive types") { + checkType(Add(d1, i), DecimalType(12, 1)) + checkType(Add(d1, f), DoubleType) + checkType(Add(i, d1), DecimalType(12, 1)) + checkType(Add(f, d1), DoubleType) + checkType(Add(d1, Cast(i, LongType)), DecimalType(22, 1)) + checkType(Add(d1, Cast(i, ShortType)), DecimalType(7, 1)) + checkType(Add(d1, Cast(i, ByteType)), DecimalType(5, 1)) + checkType(Add(d1, Cast(i, DoubleType)), DoubleType) + } + + test("maximum decimals") { + for (expr <- Seq(d1, d2, i, u)) { + checkType(Add(expr, u), DecimalType(38, 17)) + checkType(Subtract(expr, u), DecimalType(38, 17)) + } + + checkType(Multiply(d1, u), DecimalType(38, 16)) + checkType(Multiply(d2, u), DecimalType(38, 14)) + checkType(Multiply(i, u), DecimalType(38, 7)) + checkType(Multiply(u, u), DecimalType(38, 6)) + + checkType(Divide(u, d1), DecimalType(38, 17)) + checkType(Divide(u, d2), DecimalType(38, 16)) + checkType(Divide(u, i), DecimalType(38, 18)) + checkType(Divide(u, u), DecimalType(38, 6)) + + for (expr <- Seq(f, b)) { + checkType(Add(expr, u), DoubleType) + checkType(Subtract(expr, u), DoubleType) + checkType(Multiply(expr, u), DoubleType) + checkType(Divide(expr, u), DoubleType) + } + } +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 37e4c68f70f5..069d697bd454 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -639,6 +639,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeGlutenTest("to_unix_timestamp") .excludeGlutenTest("Hour") enableSuite[GlutenDecimalExpressionSuite].exclude("MakeDecimal") + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] .exclude("sha2") .exclude("murmur3/xxHash64/hive hash: struct") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e6e42acb31a2..40ecc3c351e6 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -19,7 +19,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} +import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} import org.apache.spark.sql.connector._ import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} import org.apache.spark.sql.execution._ @@ -122,6 +122,7 @@ class VeloxTestSettings extends BackendTestSettings { // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") enableSuite[GlutenDecimalExpressionSuite] + enableSuite[GlutenDecimalPrecisionSuite] enableSuite[GlutenHashExpressionsSuite] enableSuite[GlutenHigherOrderFunctionsSuite] enableSuite[GlutenIntervalExpressionsSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala new file mode 100644 index 000000000000..97e752d7d046 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDecimalPrecisionSuite.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.gluten.expression._ + +import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} +import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project} +import org.apache.spark.sql.types._ + +class GlutenDecimalPrecisionSuite extends GlutenTestsTrait { + private val catalog = new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry) + private val analyzer = new Analyzer(catalog) + + private val relation = LocalRelation( + AttributeReference("i", IntegerType)(), + AttributeReference("d1", DecimalType(2, 1))(), + AttributeReference("d2", DecimalType(5, 2))(), + AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(), + AttributeReference("f", FloatType)(), + AttributeReference("b", DoubleType)() + ) + + private val i: Expression = UnresolvedAttribute("i") + private val d1: Expression = UnresolvedAttribute("d1") + private val d2: Expression = UnresolvedAttribute("d2") + private val u: Expression = UnresolvedAttribute("u") + private val f: Expression = UnresolvedAttribute("f") + private val b: Expression = UnresolvedAttribute("b") + + private def checkType(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Seq(Alias(expression, "c")()), relation)) + assert(plan.isInstanceOf[Project]) + val expr = plan.asInstanceOf[Project].projectList.head + assert(expr.dataType == expectedType) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.dataType == expectedType) + } + + private def stripAlias(expr: Expression): Expression = { + expr match { + case a: Alias => stripAlias(a.child) + case _ => expr + } + } + + private def checkComparison(expression: Expression, expectedType: DataType): Unit = { + val plan = analyzer.execute(Project(Alias(expression, "c")() :: Nil, relation)) + assert(plan.isInstanceOf[Project]) + val expr = stripAlias(plan.asInstanceOf[Project].projectList.head) + val transformedExpr = + ExpressionConverter.replaceWithExpressionTransformer(expr, plan.inputSet.toSeq) + assert(transformedExpr.isInstanceOf[GenericExpressionTransformer]) + val binaryComparison = transformedExpr.asInstanceOf[GenericExpressionTransformer] + assert(binaryComparison.original.isInstanceOf[BinaryComparison]) + assert(binaryComparison.children.size == 2) + assert(binaryComparison.children.forall(_.dataType == expectedType)) + } + + test("basic operations") { + checkType(Add(d1, d2), DecimalType(6, 2)) + checkType(Subtract(d1, d2), DecimalType(6, 2)) + checkType(Multiply(d1, d2), DecimalType(8, 3)) + checkType(Divide(d1, d2), DecimalType(10, 7)) + checkType(Divide(d2, d1), DecimalType(10, 6)) + + checkType(Add(Add(d1, d2), d1), DecimalType(7, 2)) + checkType(Add(Add(d1, d1), d1), DecimalType(4, 1)) + checkType(Add(d1, Add(d1, d1)), DecimalType(4, 1)) + checkType(Add(Add(Add(d1, d2), d1), d2), DecimalType(8, 2)) + checkType(Add(Add(d1, d2), Add(d1, d2)), DecimalType(7, 2)) + checkType(Subtract(Subtract(d2, d1), d1), DecimalType(7, 2)) + checkType(Multiply(Multiply(d1, d1), d2), DecimalType(11, 4)) + checkType(Divide(d2, Add(d1, d1)), DecimalType(10, 6)) + } + + test("Comparison operations") { + checkComparison(EqualTo(i, d1), DecimalType(11, 1)) + checkComparison(EqualNullSafe(d2, d1), DecimalType(5, 2)) + checkComparison(LessThan(i, d1), DecimalType(11, 1)) + checkComparison(LessThanOrEqual(d1, d2), DecimalType(5, 2)) + checkComparison(GreaterThan(d2, u), DecimalType.SYSTEM_DEFAULT) + checkComparison(GreaterThanOrEqual(d1, f), DoubleType) + checkComparison(GreaterThan(d2, d2), DecimalType(5, 2)) + } + + test("bringing in primitive types") { + checkType(Add(d1, i), DecimalType(12, 1)) + checkType(Add(d1, f), DoubleType) + checkType(Add(i, d1), DecimalType(12, 1)) + checkType(Add(f, d1), DoubleType) + checkType(Add(d1, Cast(i, LongType)), DecimalType(22, 1)) + checkType(Add(d1, Cast(i, ShortType)), DecimalType(7, 1)) + checkType(Add(d1, Cast(i, ByteType)), DecimalType(5, 1)) + checkType(Add(d1, Cast(i, DoubleType)), DoubleType) + } + + test("maximum decimals") { + for (expr <- Seq(d1, d2, i, u)) { + checkType(Add(expr, u), DecimalType(38, 17)) + checkType(Subtract(expr, u), DecimalType(38, 17)) + } + + checkType(Multiply(d1, u), DecimalType(38, 16)) + checkType(Multiply(d2, u), DecimalType(38, 14)) + checkType(Multiply(i, u), DecimalType(38, 7)) + checkType(Multiply(u, u), DecimalType(38, 6)) + + checkType(Divide(u, d1), DecimalType(38, 17)) + checkType(Divide(u, d2), DecimalType(38, 16)) + checkType(Divide(u, i), DecimalType(38, 18)) + checkType(Divide(u, u), DecimalType(38, 6)) + + for (expr <- Seq(f, b)) { + checkType(Add(expr, u), DoubleType) + checkType(Subtract(expr, u), DoubleType) + checkType(Multiply(expr, u), DoubleType) + checkType(Divide(expr, u), DoubleType) + } + } +} From 70ee0f411cde6d6bbf3772ea3d623ef698b9f174 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Fri, 17 May 2024 16:28:35 +0800 Subject: [PATCH 094/402] [VL] Add BufferedOutputStream to track the memory usage in PrestoSerializer (#5785) --- cpp/velox/CMakeLists.txt | 1 + cpp/velox/memory/BufferOutputStream.cc | 45 ++++++++++++ cpp/velox/memory/BufferOutputStream.h | 42 +++++++++++ .../shuffle/VeloxSortBasedShuffleWriter.cc | 25 +++---- .../shuffle/VeloxSortBasedShuffleWriter.h | 8 +-- cpp/velox/tests/BufferOutputStreamTest.cc | 70 +++++++++++++++++++ cpp/velox/tests/CMakeLists.txt | 1 + 7 files changed, 171 insertions(+), 21 deletions(-) create mode 100644 cpp/velox/memory/BufferOutputStream.cc create mode 100644 cpp/velox/memory/BufferOutputStream.h create mode 100644 cpp/velox/tests/BufferOutputStreamTest.cc diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index c058883b603f..9bedfe45ba0e 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -300,6 +300,7 @@ set(VELOX_SRCS jni/VeloxJniWrapper.cc jni/JniFileSystem.cc jni/JniUdf.cc + memory/BufferOutputStream.cc memory/VeloxColumnarBatch.cc memory/VeloxMemoryManager.cc operators/functions/RegistrationAllFunctions.cc diff --git a/cpp/velox/memory/BufferOutputStream.cc b/cpp/velox/memory/BufferOutputStream.cc new file mode 100644 index 000000000000..31d7b0936545 --- /dev/null +++ b/cpp/velox/memory/BufferOutputStream.cc @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "memory/BufferOutputStream.h" + +namespace gluten { +BufferOutputStream::BufferOutputStream( + facebook::velox::memory::MemoryPool* pool, + int32_t initialSize, + facebook::velox::OutputStreamListener* listener) + : facebook::velox::OutputStream(listener) { + buffer_ = facebook::velox::AlignedBuffer::allocate(initialSize, pool); + buffer_->setSize(0); +} + +void BufferOutputStream::write(const char* s, std::streamsize count) { + facebook::velox::AlignedBuffer::appendTo(&buffer_, s, count); +} + +std::streampos BufferOutputStream::tellp() const { + return buffer_->size(); +} + +void BufferOutputStream::seekp(std::streampos pos) { + buffer_->setSize(pos); +} + +facebook::velox::BufferPtr BufferOutputStream::getBuffer() const { + return buffer_; +} +} // namespace gluten diff --git a/cpp/velox/memory/BufferOutputStream.h b/cpp/velox/memory/BufferOutputStream.h new file mode 100644 index 000000000000..49774e09de73 --- /dev/null +++ b/cpp/velox/memory/BufferOutputStream.h @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/buffer/Buffer.h" +#include "velox/common/memory/ByteStream.h" + +namespace gluten { +class BufferOutputStream : public facebook::velox::OutputStream { + public: + BufferOutputStream( + facebook::velox::memory::MemoryPool* pool, + int32_t initialSize = facebook::velox::memory::AllocationTraits::kPageSize, + facebook::velox::OutputStreamListener* listener = nullptr); + + void write(const char* s, std::streamsize count); + + std::streampos tellp() const; + + void seekp(std::streampos pos); + + facebook::velox::BufferPtr getBuffer() const; + + private: + facebook::velox::BufferPtr buffer_; +}; +} // namespace gluten diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc index b0c2cc8adc7b..bd56bc62e8f0 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc @@ -84,6 +84,7 @@ arrow::Status VeloxSortBasedShuffleWriter::init() { for (auto pid = 0; pid < numPartitions_; ++pid) { rowVectorIndexMap_[pid].reserve(options_.bufferSize); } + bufferOutputStream_ = std::make_unique(veloxPool_.get()); return arrow::Status::OK(); } @@ -153,18 +154,12 @@ arrow::Status VeloxSortBasedShuffleWriter::write(std::shared_ptr return arrow::Status::OK(); } -arrow::Status VeloxSortBasedShuffleWriter::evictBatch( - uint32_t partitionId, - std::ostringstream* output, - facebook::velox::OStreamOutputStream* out, - facebook::velox::RowTypePtr* rowTypePtr) { +arrow::Status VeloxSortBasedShuffleWriter::evictBatch(uint32_t partitionId, facebook::velox::RowTypePtr* rowTypePtr) { int64_t rawSize = batch_->size(); - batch_->flush(out); - const std::string& outputStr = output->str(); - RETURN_NOT_OK(partitionWriter_->evict(partitionId, rawSize, outputStr.c_str(), outputStr.size())); - batch_.reset(); - output->clear(); - output->str(""); + bufferOutputStream_->seekp(0); + batch_->flush(bufferOutputStream_.get()); + auto buffer = bufferOutputStream_->getBuffer(); + RETURN_NOT_OK(partitionWriter_->evict(partitionId, rawSize, buffer->as(), buffer->size())); batch_ = std::make_unique(veloxPool_.get(), serde_.get()); batch_->createStreamTree(*rowTypePtr, options_.bufferSize, &serdeOptions_); return arrow::Status::OK(); @@ -174,8 +169,6 @@ arrow::Status VeloxSortBasedShuffleWriter::evictRowVector(uint32_t partitionId) int32_t rowNum = 0; const int32_t maxBatchNum = options_.bufferSize; auto rowTypePtr = std::static_pointer_cast(rowType_.value()); - std::ostringstream output; - facebook::velox::OStreamOutputStream out(&output); if (options_.partitioning != Partitioning::kSingle) { if (auto it = rowVectorIndexMap_.find(partitionId); it != rowVectorIndexMap_.end()) { @@ -219,7 +212,7 @@ arrow::Status VeloxSortBasedShuffleWriter::evictRowVector(uint32_t partitionId) rowNum += groupedSize[pair.first]; if (rowNum >= maxBatchNum) { rowNum = 0; - RETURN_NOT_OK(evictBatch(partitionId, &output, &out, &rowTypePtr)); + RETURN_NOT_OK(evictBatch(partitionId, &rowTypePtr)); } } @@ -231,13 +224,13 @@ arrow::Status VeloxSortBasedShuffleWriter::evictRowVector(uint32_t partitionId) rowNum += rowVectorPtr->size(); batch_->append(rowVectorPtr); if (rowNum >= maxBatchNum) { - RETURN_NOT_OK(evictBatch(partitionId, &output, &out, &rowTypePtr)); + RETURN_NOT_OK(evictBatch(partitionId, &rowTypePtr)); rowNum = 0; } } } if (rowNum > 0) { - RETURN_NOT_OK(evictBatch(partitionId, &output, &out, &rowTypePtr)); + RETURN_NOT_OK(evictBatch(partitionId, &rowTypePtr)); } return arrow::Status::OK(); } diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h index e3ac07dfcd82..710590184f9b 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h @@ -37,6 +37,7 @@ #include #include "VeloxShuffleWriter.h" +#include "memory/BufferOutputStream.h" #include "memory/VeloxMemoryManager.h" #include "shuffle/PartitionWriter.h" #include "shuffle/Partitioner.h" @@ -65,11 +66,7 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { arrow::Status evictRowVector(uint32_t partitionId) override; - arrow::Status evictBatch( - uint32_t partitionId, - std::ostringstream* output, - facebook::velox::OStreamOutputStream* out, - facebook::velox::RowTypePtr* rowTypePtr); + arrow::Status evictBatch(uint32_t partitionId, facebook::velox::RowTypePtr* rowTypePtr); private: VeloxSortBasedShuffleWriter( @@ -93,6 +90,7 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { std::optional rowType_; std::unique_ptr batch_; + std::unique_ptr bufferOutputStream_; // Partition ID -> Row Count // subscript: Partition ID diff --git a/cpp/velox/tests/BufferOutputStreamTest.cc b/cpp/velox/tests/BufferOutputStreamTest.cc new file mode 100644 index 000000000000..3b3f78ceaefe --- /dev/null +++ b/cpp/velox/tests/BufferOutputStreamTest.cc @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "memory/BufferOutputStream.h" +#include "memory/VeloxColumnarBatch.h" +#include "velox/common/memory/ByteStream.h" +#include "velox/vector/tests/utils/VectorTestBase.h" + +using namespace facebook::velox; + +namespace gluten { +class BufferOutputStreamTest : public ::testing::Test, public test::VectorTestBase { + protected: + // Velox requires the mem manager to be instanced. + static void SetUpTestCase() { + memory::MemoryManager::testingSetInstance({}); + } + + std::shared_ptr veloxPool_ = defaultLeafVeloxMemoryPool(); +}; + +TEST_F(BufferOutputStreamTest, outputStream) { + auto out = std::make_unique(veloxPool_.get(), 10000); + std::stringstream referenceSStream; + auto reference = std::make_unique(&referenceSStream); + for (auto i = 0; i < 100; ++i) { + std::string data; + data.resize(10000); + std::fill(data.begin(), data.end(), i); + out->write(data.data(), data.size()); + reference->write(data.data(), data.size()); + } + EXPECT_EQ(reference->tellp(), out->tellp()); + for (auto i = 0; i < 100; ++i) { + std::string data; + data.resize(6000); + std::fill(data.begin(), data.end(), i + 10); + out->seekp(i * 10000 + 5000); + reference->seekp(i * 10000 + 5000); + out->write(data.data(), data.size()); + reference->write(data.data(), data.size()); + } + auto str = referenceSStream.str(); + auto numBytes = veloxPool_->currentBytes(); + EXPECT_LT(0, numBytes); + { + auto buffer = out->getBuffer(); + EXPECT_EQ(numBytes, veloxPool_->currentBytes()); + EXPECT_EQ(str, std::string(buffer->as(), buffer->size())); + } + + out.reset(); + // We expect dropping the stream frees the backing memory. + EXPECT_EQ(0, veloxPool_->currentBytes()); +} +} // namespace gluten diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt index 58482fe1564e..a5bd5b4f7c9d 100644 --- a/cpp/velox/tests/CMakeLists.txt +++ b/cpp/velox/tests/CMakeLists.txt @@ -61,3 +61,4 @@ add_velox_test( FilePathGenerator.cc) add_velox_test(spark_functions_test SOURCES SparkFunctionTest.cc) add_velox_test(execution_ctx_test SOURCES RuntimeTest.cc) +add_velox_test(buffer_outputstream_test SOURCES BufferOutputStreamTest.cc) From 038d9cbaa840a7c9aea5324e510dffaace4a8801 Mon Sep 17 00:00:00 2001 From: zhaokuo Date: Fri, 17 May 2024 16:56:21 +0800 Subject: [PATCH 095/402] [VL] Use MemConfig to replace MemConfigMutable to make the code cleaner and to improve performance (#5784) --- cpp/velox/compute/VeloxBackend.cc | 57 ++++++++----------- cpp/velox/compute/VeloxBackend.h | 6 +- cpp/velox/compute/VeloxRuntime.cc | 12 ++-- cpp/velox/compute/VeloxRuntime.h | 2 +- cpp/velox/compute/WholeStageResultIterator.cc | 2 +- cpp/velox/compute/WholeStageResultIterator.h | 2 +- .../writer/VeloxParquetDatasourceABFS.h | 6 +- .../writer/VeloxParquetDatasourceHDFS.h | 8 +-- .../writer/VeloxParquetDatasourceS3.h | 8 +-- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 3 +- cpp/velox/utils/ConfigExtractor.cc | 37 ++++++------ cpp/velox/utils/ConfigExtractor.h | 3 +- 12 files changed, 61 insertions(+), 85 deletions(-) diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index b2fb1c964e22..187c36e1e8bd 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -61,13 +61,13 @@ gluten::Runtime* veloxRuntimeFactory(const std::unordered_map& conf) { - backendConf_ = std::make_shared(conf); + backendConf_ = std::make_shared(conf); // Register Velox runtime factory gluten::Runtime::registerFactory(gluten::kVeloxRuntimeKind, veloxRuntimeFactory); if (backendConf_->get(kDebugModeEnabled, false)) { - LOG(INFO) << "VeloxBackend config:" << printConfig(backendConf_->valuesCopy()); + LOG(INFO) << "VeloxBackend config:" << printConfig(backendConf_->values()); } // Init glog and log level. @@ -188,46 +188,39 @@ void VeloxBackend::initCache() { void VeloxBackend::initConnector() { // The configs below are used at process level. - auto connectorConf = std::make_shared(backendConf_->valuesCopy()); + std::unordered_map connectorConfMap = backendConf_->values(); auto hiveConf = getHiveConfig(backendConf_); for (auto& [k, v] : hiveConf->valuesCopy()) { - connectorConf->setValue(k, v); + connectorConfMap[k] = v; } #ifdef ENABLE_ABFS - const auto& confValue = backendConf_->valuesCopy(); + const auto& confValue = backendConf_->values(); for (auto& [k, v] : confValue) { if (k.find("fs.azure.account.key") == 0) { - connectorConf->setValue(k, v); + connectorConfMap[k] = v; } else if (k.find("spark.hadoop.fs.azure.account.key") == 0) { constexpr int32_t accountKeyPrefixLength = 13; - connectorConf->setValue(k.substr(accountKeyPrefixLength), v); + connectorConfMap[k.substr(accountKeyPrefixLength)] = v; } } #endif - - connectorConf->setValue( - velox::connector::hive::HiveConfig::kEnableFileHandleCache, - backendConf_->get(kVeloxFileHandleCacheEnabled, kVeloxFileHandleCacheEnabledDefault) ? "true" : "false"); - - connectorConf->setValue( - velox::connector::hive::HiveConfig::kMaxCoalescedBytes, - backendConf_->get(kMaxCoalescedBytes, "67108864")); // 64M - connectorConf->setValue( - velox::connector::hive::HiveConfig::kMaxCoalescedDistanceBytes, - backendConf_->get(kMaxCoalescedDistanceBytes, "1048576")); // 1M - connectorConf->setValue( - velox::connector::hive::HiveConfig::kPrefetchRowGroups, backendConf_->get(kPrefetchRowGroups, "1")); - connectorConf->setValue( - velox::connector::hive::HiveConfig::kLoadQuantum, - backendConf_->get(kLoadQuantum, "268435456")); // 256M - connectorConf->setValue( - velox::connector::hive::HiveConfig::kFooterEstimatedSize, - backendConf_->get(kDirectorySizeGuess, "32768")); // 32K - connectorConf->setValue( - velox::connector::hive::HiveConfig::kFilePreloadThreshold, - backendConf_->get(kFilePreloadThreshold, "1048576")); // 1M + connectorConfMap[velox::connector::hive::HiveConfig::kEnableFileHandleCache] = + backendConf_->get(kVeloxFileHandleCacheEnabled, kVeloxFileHandleCacheEnabledDefault) ? "true" : "false"; + + connectorConfMap[velox::connector::hive::HiveConfig::kMaxCoalescedBytes] = + backendConf_->get(kMaxCoalescedBytes, "67108864"); // 64M + connectorConfMap[velox::connector::hive::HiveConfig::kMaxCoalescedDistanceBytes] = + backendConf_->get(kMaxCoalescedDistanceBytes, "1048576"); // 1M + connectorConfMap[velox::connector::hive::HiveConfig::kPrefetchRowGroups] = + backendConf_->get(kPrefetchRowGroups, "1"); + connectorConfMap[velox::connector::hive::HiveConfig::kLoadQuantum] = + backendConf_->get(kLoadQuantum, "268435456"); // 256M + connectorConfMap[velox::connector::hive::HiveConfig::kFooterEstimatedSize] = + backendConf_->get(kDirectorySizeGuess, "32768"); // 32K + connectorConfMap[velox::connector::hive::HiveConfig::kFilePreloadThreshold] = + backendConf_->get(kFilePreloadThreshold, "1048576"); // 1M // set cache_prefetch_min_pct default as 0 to force all loads are prefetched in DirectBufferInput. FLAGS_cache_prefetch_min_pct = backendConf_->get(kCachePrefetchMinPct, 0); @@ -238,7 +231,7 @@ void VeloxBackend::initConnector() { } velox::connector::registerConnector(std::make_shared( kHiveConnectorId, - std::make_shared(connectorConf->valuesCopy()), + std::make_shared(std::move(connectorConfMap)), ioExecutor_.get())); } @@ -265,8 +258,4 @@ VeloxBackend* VeloxBackend::get() { return instance_.get(); } -const std::shared_ptr VeloxBackend::getBackendConf() const { - return backendConf_; -} - } // namespace gluten diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h index 891bdd2cc408..e8298eeed192 100644 --- a/cpp/velox/compute/VeloxBackend.h +++ b/cpp/velox/compute/VeloxBackend.h @@ -53,7 +53,9 @@ class VeloxBackend { facebook::velox::cache::AsyncDataCache* getAsyncDataCache() const; - const std::shared_ptr getBackendConf() const; + std::shared_ptr getBackendConf() const { + return backendConf_; + } void tearDown() { // Destruct IOThreadPoolExecutor will join all threads. @@ -90,7 +92,7 @@ class VeloxBackend { std::string cachePathPrefix_; std::string cacheFilePrefix_; - std::shared_ptr backendConf_; + std::shared_ptr backendConf_; }; } // namespace gluten diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index 15c84b41cdad..44f04ef31ae7 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -56,7 +56,7 @@ namespace gluten { VeloxRuntime::VeloxRuntime(const std::unordered_map& confMap) : Runtime(confMap) { // Refresh session config. - veloxCfg_ = std::make_shared(confMap_); + veloxCfg_ = std::make_shared(confMap_); debugModeEnabled_ = veloxCfg_->get(kDebugModeEnabled, false); FLAGS_minloglevel = veloxCfg_->get(kGlogSeverityLevel, FLAGS_minloglevel); FLAGS_v = veloxCfg_->get(kGlogVerboseLevel, FLAGS_v); @@ -275,11 +275,11 @@ std::unique_ptr VeloxRuntime::createColumnarBatchSerial } void VeloxRuntime::dumpConf(const std::string& path) { - auto backendConf = VeloxBackend::get()->getBackendConf()->valuesCopy(); - auto allConf = backendConf; + const auto& backendConfMap = VeloxBackend::get()->getBackendConf()->values(); + auto allConfMap = backendConfMap; for (const auto& pair : confMap_) { - allConf.insert_or_assign(pair.first, pair.second); + allConfMap.insert_or_assign(pair.first, pair.second); } // Open file "velox.conf" for writing, automatically creating it if it doesn't exist, @@ -292,13 +292,13 @@ void VeloxRuntime::dumpConf(const std::string& path) { // Calculate the maximum key length for alignment. size_t maxKeyLength = 0; - for (const auto& pair : allConf) { + for (const auto& pair : allConfMap) { maxKeyLength = std::max(maxKeyLength, pair.first.length()); } // Write each key-value pair to the file with adjusted spacing for alignment outFile << "[Backend Conf]" << std::endl; - for (const auto& pair : backendConf) { + for (const auto& pair : backendConfMap) { outFile << std::left << std::setw(maxKeyLength + 1) << pair.first << ' ' << pair.second << std::endl; } outFile << std::endl << "[Session Conf]" << std::endl; diff --git a/cpp/velox/compute/VeloxRuntime.h b/cpp/velox/compute/VeloxRuntime.h index e2097edb1b7c..80408bccb2b5 100644 --- a/cpp/velox/compute/VeloxRuntime.h +++ b/cpp/velox/compute/VeloxRuntime.h @@ -132,7 +132,7 @@ class VeloxRuntime final : public Runtime { private: std::shared_ptr veloxPlan_; - std::shared_ptr veloxCfg_; + std::shared_ptr veloxCfg_; bool debugModeEnabled_{false}; std::unordered_map> emptySchemaBatchLoopUp_; diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 006b37588005..06a7a7c391ab 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -62,7 +62,7 @@ WholeStageResultIterator::WholeStageResultIterator( const std::unordered_map& confMap, const SparkTaskInfo& taskInfo) : memoryManager_(memoryManager), - veloxCfg_(std::make_shared(confMap)), + veloxCfg_(std::make_shared(confMap)), taskInfo_(taskInfo), veloxPlan_(planNode), scanNodeIds_(scanNodeIds), diff --git a/cpp/velox/compute/WholeStageResultIterator.h b/cpp/velox/compute/WholeStageResultIterator.h index 10c1937b78ef..0ad3877ff203 100644 --- a/cpp/velox/compute/WholeStageResultIterator.h +++ b/cpp/velox/compute/WholeStageResultIterator.h @@ -103,7 +103,7 @@ class WholeStageResultIterator : public ColumnarBatchIterator { VeloxMemoryManager* memoryManager_; /// Config, task and plan. - const std::shared_ptr veloxCfg_; + std::shared_ptr veloxCfg_; const SparkTaskInfo taskInfo_; std::shared_ptr task_; std::shared_ptr veloxPlan_; diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h index 2251a46ffa69..208e6a7ec55c 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h @@ -42,10 +42,8 @@ class VeloxParquetDatasourceABFS final : public VeloxParquetDatasource { std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} void init(const std::unordered_map& sparkConfs) override { - auto confs = std::make_shared(sparkConfs); - auto hiveConfs = getHiveConfig(confs); - auto fileSystem = filesystems::getFileSystem( - filePath_, std::make_shared(hiveConfs->valuesCopy())); + auto hiveConf = getHiveConfig(std::make_shared(sparkConfs)); + auto fileSystem = filesystems::getFileSystem(filePath_, hiveConf); auto* abfsFileSystem = dynamic_cast(fileSystem.get()); sink_ = std::make_unique( abfsFileSystem->openFileForWrite(filePath_, {{}, sinkPool_.get()}), filePath_); diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h b/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h index 1b37a7c6fb0a..32cf960cbf2f 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h @@ -42,12 +42,8 @@ class VeloxParquetDatasourceHDFS final : public VeloxParquetDatasource { std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} void init(const std::unordered_map& sparkConfs) override { - auto confs = std::make_shared(sparkConfs); - auto hiveConfs = getHiveConfig(confs); - sink_ = dwio::common::FileSink::create( - filePath_, - {.connectorProperties = std::make_shared(hiveConfs->valuesCopy()), - .pool = sinkPool_.get()}); + auto hiveConf = getHiveConfig(std::make_shared(sparkConfs)); + sink_ = dwio::common::FileSink::create(filePath_, {.connectorProperties = hiveConf, .pool = sinkPool_.get()}); VeloxParquetDatasource::init(sparkConfs); } }; diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h b/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h index 92965d4e36dd..a5c49fcd9f81 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h @@ -42,12 +42,8 @@ class VeloxParquetDatasourceS3 final : public VeloxParquetDatasource { std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} void init(const std::unordered_map& sparkConfs) override { - auto confs = std::make_shared(sparkConfs); - auto hiveConfs = getHiveConfig(confs); - sink_ = dwio::common::FileSink::create( - filePath_, - {.connectorProperties = std::make_shared(hiveConfs->valuesCopy()), - .pool = sinkPool_.get()}); + auto hiveConf = getHiveConfig(std::make_shared(sparkConfs)); + sink_ = dwio::common::FileSink::create(filePath_, {.connectorProperties = hiveConf, .pool = sinkPool_.get()}); VeloxParquetDatasource::init(sparkConfs); } }; diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 366ab5abdc9d..b50f9bd346be 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -1076,8 +1076,7 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: std::vector veloxTypeList; std::vector columnTypes; // Convert field names into lower case when not case-sensitive. - std::shared_ptr veloxCfg = - std::make_shared(confMap_); + std::unique_ptr veloxCfg = std::make_unique(confMap_); bool asLowerCase = !veloxCfg->get(kCaseSensitive, false); if (readRel.has_base_schema()) { const auto& baseSchema = readRel.base_schema(); diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index 3889fee6ad76..0cbba37a783d 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -52,9 +52,8 @@ std::string getConfigValue( return got->second; } -std::shared_ptr getHiveConfig( - const std::shared_ptr& conf) { - auto hiveConf = std::make_shared(); +std::shared_ptr getHiveConfig(std::shared_ptr conf) { + std::unordered_map hiveConfMap; #ifdef ENABLE_S3 std::string awsAccessKey = conf->get("spark.hadoop.fs.s3a.access.key", ""); @@ -82,24 +81,23 @@ std::shared_ptr getHiveConfig( } if (useInstanceCredentials) { - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3UseInstanceCredentials, "true"); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3UseInstanceCredentials] = "true"; } else if (!iamRole.empty()) { - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3IamRole, iamRole); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3IamRole] = iamRole; if (!iamRoleSessionName.empty()) { - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3IamRoleSessionName, iamRoleSessionName); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3IamRoleSessionName] = iamRoleSessionName; } } else { - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3AwsAccessKey, awsAccessKey); - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3AwsSecretKey, awsSecretKey); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3AwsAccessKey] = awsAccessKey; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3AwsSecretKey] = awsSecretKey; } // Only need to set s3 endpoint when not use instance credentials. if (!useInstanceCredentials) { - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3Endpoint, awsEndpoint); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3Endpoint] = awsEndpoint; } - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3SSLEnabled, sslEnabled ? "true" : "false"); - hiveConf->setValue( - facebook::velox::connector::hive::HiveConfig::kS3PathStyleAccess, pathStyleAccess ? "true" : "false"); - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kS3LogLevel, awsSdkLogLevel); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3SSLEnabled] = sslEnabled ? "true" : "false"; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3PathStyleAccess] = pathStyleAccess ? "true" : "false"; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3LogLevel] = awsSdkLogLevel; #endif #ifdef ENABLE_GCS @@ -118,8 +116,8 @@ std::shared_ptr getHiveConfig( } if (!gcsEndpoint.empty() && !gcsScheme.empty()) { - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kGCSScheme, gcsScheme); - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kGCSEndpoint, gcsEndpoint); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSScheme] = gcsScheme; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSEndpoint] = gcsEndpoint; } } @@ -133,7 +131,7 @@ std::shared_ptr getHiveConfig( auto stream = std::ifstream(gsAuthServiceAccountJsonKeyfile.value()); stream.exceptions(std::ios::badbit); std::string gsAuthServiceAccountJson = std::string(std::istreambuf_iterator(stream.rdbuf()), {}); - hiveConf->setValue(facebook::velox::connector::hive::HiveConfig::kGCSCredentials, gsAuthServiceAccountJson); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSCredentials] = gsAuthServiceAccountJson; } else { LOG(WARNING) << "STARTUP: conf spark.hadoop.fs.gs.auth.type is set to SERVICE_ACCOUNT_JSON_KEYFILE, " "however conf spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set"; @@ -143,11 +141,10 @@ std::shared_ptr getHiveConfig( } #endif - hiveConf->setValue( - facebook::velox::connector::hive::HiveConfig::kEnableFileHandleCache, - conf->get(kVeloxFileHandleCacheEnabled, kVeloxFileHandleCacheEnabledDefault) ? "true" : "false"); + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kEnableFileHandleCache] = + conf->get(kVeloxFileHandleCacheEnabled, kVeloxFileHandleCacheEnabledDefault) ? "true" : "false"; - return hiveConf; + return std::make_shared(std::move(hiveConfMap)); } } // namespace gluten diff --git a/cpp/velox/utils/ConfigExtractor.h b/cpp/velox/utils/ConfigExtractor.h index 09b1178e694a..c5f662c950de 100644 --- a/cpp/velox/utils/ConfigExtractor.h +++ b/cpp/velox/utils/ConfigExtractor.h @@ -33,7 +33,6 @@ std::string getConfigValue( const std::string& key, const std::optional& fallbackValue); -std::shared_ptr getHiveConfig( - const std::shared_ptr& conf); +std::shared_ptr getHiveConfig(std::shared_ptr conf); } // namespace gluten From 30cc3b6cffaba83f1e8be6220760af40d32223c2 Mon Sep 17 00:00:00 2001 From: zhaokuo Date: Fri, 17 May 2024 18:54:02 +0800 Subject: [PATCH 096/402] fix warnings by -Wunused-but-set-variable --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index b50f9bd346be..34ba6057c15f 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -2158,8 +2158,8 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( upperBound = getMax(); } - bool lowerUnbounded = true; - bool upperUnbounded = true; + [[maybe_unused]] bool lowerUnbounded = true; + [[maybe_unused]] bool upperUnbounded = true; bool lowerExclusive = false; bool upperExclusive = false; From aa9221af8f8e7784caa5311c48ac0a47f67e8cfd Mon Sep 17 00:00:00 2001 From: zhaokuo Date: Fri, 17 May 2024 19:01:32 +0800 Subject: [PATCH 097/402] Revert "fix warnings by -Wunused-but-set-variable" This reverts commit 30cc3b6cffaba83f1e8be6220760af40d32223c2. --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 34ba6057c15f..b50f9bd346be 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -2158,8 +2158,8 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( upperBound = getMax(); } - [[maybe_unused]] bool lowerUnbounded = true; - [[maybe_unused]] bool upperUnbounded = true; + bool lowerUnbounded = true; + bool upperUnbounded = true; bool lowerExclusive = false; bool upperExclusive = false; From 1454aaafe4202ceff5010bf7376ac09d67de31ce Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 17 May 2024 20:41:20 +0800 Subject: [PATCH 098/402] [VL][CI] Cache native libraries to re-use them in Spark test jobs (#5768) Co-authored-by: Yuan Zhou --- .github/workflows/velox_docker.yml | 291 +++++++++++++++++------------ 1 file changed, 174 insertions(+), 117 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index a92abda3c4ab..b7ee3be4145e 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -47,7 +47,7 @@ concurrency: cancel-in-progress: true jobs: - build-native-lib: + build-native-lib-centos-7: runs-on: ubuntu-20.04 container: apache/gluten:gluten-vcpkg-builder_2024_03_17 # centos7 with dependencies installed steps: @@ -68,10 +68,10 @@ jobs: - uses: actions/upload-artifact@v2 with: path: ./cpp/build/releases/ - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} run-tpc-test-ubuntu: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -95,7 +95,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Setup java and maven run: | @@ -119,7 +119,7 @@ jobs: --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 run-tpc-test-centos: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -143,7 +143,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Update mirror list if: matrix.os == 'centos:8' @@ -195,7 +195,7 @@ jobs: --extra-conf=spark.gluten.sql.ras.enabled=true run-tpc-test-ubuntu-oom: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -218,7 +218,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Setup java and maven run: | @@ -292,7 +292,7 @@ jobs: -d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g || true run-tpc-test-ubuntu-randomkill: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -315,7 +315,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Setup java and maven run: | @@ -339,7 +339,7 @@ jobs: --skip-data-gen --random-kill-tasks run-tpc-test-ubuntu-sf30: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -363,7 +363,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Setup java and maven run: | @@ -391,7 +391,7 @@ jobs: --skip-data-gen --shard=${{ matrix.shard }} run-tpc-test-centos8-uniffle: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -403,7 +403,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Update mirror list run: | @@ -451,7 +451,7 @@ jobs: --local --preset=velox-with-uniffle --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 run-tpc-test-ubuntu-2204-celeborn: - needs: build-native-lib + needs: build-native-lib-centos-7 strategy: fail-fast: false matrix: @@ -464,7 +464,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Setup java and maven run: | @@ -494,13 +494,74 @@ jobs: GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox-with-celeborn --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1 + build-native-lib-centos-8: + runs-on: ubuntu-20.04 + container: ghcr.io/facebookincubator/velox-dev:circleci-avx + steps: + - uses: actions/checkout@v2 + - name: Generate cache key + run: | + echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './github/workflows/*') }} > cache-key + - name: Cache + id: cache + uses: actions/cache/restore@v3 + with: + path: ./cpp/build/releases/ + key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }} + - name: Setup build dependency + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + yum install sudo patch java-1.8.0-openjdk-devel wget -y + # Required by building arrow java. + wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + - name: Build Gluten Velox third party + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + cd ep/build-velox/src + ./get_velox.sh + source /opt/rh/gcc-toolset-9/enable + ./build_arrow_deps_centos8.sh + ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON + cd $GITHUB_WORKSPACE/cpp + ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON + - uses: actions/upload-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases/ + - uses: actions/upload-artifact@v2 + with: + name: udf-example-lib-centos-8-${{github.sha}} + path: ./cpp/build/velox/udf/examples/ + - uses: actions/upload-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + run-spark-test-spark32: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download UDF Example Lib + uses: actions/download-artifact@v2 + with: + name: udf-example-lib-centos-8-${{github.sha}} + path: ./cpp/build/velox/udf/examples/ + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -517,18 +578,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - name: Gluten CPP Test run: | cd $GITHUB_WORKSPACE/cpp/build && \ @@ -562,19 +611,30 @@ jobs: with: name: golden-files-spark32 path: /tmp/tpch-approved-plan/** - - name: Gluten CPP Benchmark Test - run: | - # This test depends on example.json generated by the above mvn test. - cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ - ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 + # - name: Gluten CPP Benchmark Test + # run: | + # # This test depends on example.json generated by the above mvn test. + # cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ + # ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 run-spark-test-spark32-slow: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -591,18 +651,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON - name: Prepare spark.test.home for Spark 3.2.2 (slow tests) run: | cd $GITHUB_WORKSPACE// && \ @@ -616,12 +664,28 @@ jobs: mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark33: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download UDF Example Lib + uses: actions/download-artifact@v2 + with: + name: udf-example-lib-centos-8-${{github.sha}} + path: ./cpp/build/velox/udf/examples/ + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -638,18 +702,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - name: Prepare spark.test.home for Spark 3.3.1 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -682,12 +734,23 @@ jobs: run-spark-test-spark33-slow: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -704,18 +767,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON - name: Prepare spark.test.home for Spark 3.3.1 (slow tests) run: | cd $GITHUB_WORKSPACE// && \ @@ -729,12 +780,28 @@ jobs: mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark34: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download UDF Example Lib + uses: actions/download-artifact@v2 + with: + name: udf-example-lib-centos-8-${{github.sha}} + path: ./cpp/build/velox/udf/examples/ + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -751,18 +818,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - name: Prepare spark.test.home for Spark 3.4.2 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -795,12 +850,23 @@ jobs: run-spark-test-spark34-slow: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -817,18 +883,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON - name: Prepare spark.test.home for Spark 3.4.2 (slow tests) run: | cd $GITHUB_WORKSPACE// && \ @@ -842,12 +896,28 @@ jobs: mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark35: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download UDF Example Lib + uses: actions/download-artifact@v2 + with: + name: udf-example-lib-centos-8-${{github.sha}} + path: ./cpp/build/velox/udf/examples/ + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -864,18 +934,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - name: Prepare spark.test.home for Spark 3.5.1 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ @@ -907,12 +965,23 @@ jobs: path: /tmp/tpch-approved-plan/** run-spark-test-spark35-slow: + needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:circleci-avx env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 + - name: Download All Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-native-lib-centos-8-${{github.sha}} + path: ./cpp/build/releases + - name: Download Arrow Jars + uses: actions/download-artifact@v2 + with: + name: arrow-jars-centos-8-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -929,18 +998,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - cd ep/build-velox/src && \ - ./get_velox.sh && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./build_arrow_deps_centos8.sh && \ - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - - name: Build Gluten CPP library - run: | - cd $GITHUB_WORKSPACE/cpp && \ - source /opt/rh/gcc-toolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON - name: Prepare spark.test.home for Spark 3.5.1 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ From b6c6e264d90efc1ff268f35fad86d88cd3957465 Mon Sep 17 00:00:00 2001 From: exmy Date: Fri, 17 May 2024 20:47:11 +0800 Subject: [PATCH 099/402] [CORE] Unify the transforming for shuffle expression (#5793) --- .../backendsapi/velox/VeloxSparkPlanExecApi.scala | 10 ---------- .../apache/gluten/backendsapi/SparkPlanExecApi.scala | 7 ------- .../gluten/expression/ExpressionConverter.scala | 11 +++++------ 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index f54bf9b3f61e..cf7d38d62bbb 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -145,16 +145,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { original) } - override def genShuffleTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Shuffle): ExpressionTransformer = { - GenericExpressionTransformer( - substraitExprName, - Seq(child, LiteralTransformer(Literal(original.randomSeed.get))), - original) - } - override def genTryAddTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index aa27d1ce1865..c694d580492e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -212,13 +212,6 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(), original) } - def genShuffleTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Shuffle): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(child), original) - } - def genTryAddTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index e692890c452b..5aacbed055f4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -627,7 +627,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.function, attributeSeq, expressionsMap), a ) - case a: ArrayExists => BackendsApiManager.getSparkPlanExecApiInstance.genArrayExistsTransformer( substraitExprName, @@ -635,13 +634,13 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.function, attributeSeq, expressionsMap), a ) - case s: Shuffle => - BackendsApiManager.getSparkPlanExecApiInstance.genShuffleTransformer( + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal(s.child, attributeSeq, expressionsMap), - s - ) + Seq( + replaceWithExpressionTransformerInternal(s.child, attributeSeq, expressionsMap), + LiteralTransformer(Literal(s.randomSeed.get))), + s) case expr => GenericExpressionTransformer( substraitExprName, From b8f8154a65b02ed10980563b861bf1b273c41691 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Fri, 17 May 2024 22:26:19 +0800 Subject: [PATCH 100/402] [VL] Refine evict logic in sort shuffle writer (#5786) --- .../shuffle/VeloxSortBasedShuffleWriter.cc | 124 ++++++------------ .../shuffle/VeloxSortBasedShuffleWriter.h | 6 +- 2 files changed, 44 insertions(+), 86 deletions(-) diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc index bd56bc62e8f0..2a6bca8c0f37 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc @@ -15,47 +15,20 @@ * limitations under the License. */ -#include "VeloxSortBasedShuffleWriter.h" -#include "memory/ArrowMemory.h" +#include "shuffle/VeloxSortBasedShuffleWriter.h" #include "memory/VeloxColumnarBatch.h" #include "memory/VeloxMemoryManager.h" #include "shuffle/ShuffleSchema.h" #include "utils/Common.h" #include "utils/VeloxArrowUtils.h" #include "utils/macros.h" + #include "velox/common/base/Nulls.h" #include "velox/type/Type.h" #include "velox/vector/ComplexVector.h" -#if defined(__x86_64__) -#include -#include -#elif defined(__aarch64__) -#include -#endif - namespace gluten { -#define VELOX_SHUFFLE_WRITER_LOG_FLAG 0 - -// macro to rotate left an 8-bit value 'x' given the shift 's' is a 32-bit integer -// (x is left shifted by 's' modulo 8) OR (x right shifted by (8 - 's' modulo 8)) -#if !defined(__x86_64__) -#define rotateLeft(x, s) (x << (s - ((s >> 3) << 3)) | x >> (8 - (s - ((s >> 3) << 3)))) -#endif - -// on x86 machines, _MM_HINT_T0,T1,T2 are defined as 1, 2, 3 -// equivalent mapping to __builtin_prefetch hints is 3, 2, 1 -#if defined(__x86_64__) -#define PREFETCHT0(ptr) _mm_prefetch(ptr, _MM_HINT_T0) -#define PREFETCHT1(ptr) _mm_prefetch(ptr, _MM_HINT_T1) -#define PREFETCHT2(ptr) _mm_prefetch(ptr, _MM_HINT_T2) -#else -#define PREFETCHT0(ptr) __builtin_prefetch(ptr, 0, 3) -#define PREFETCHT1(ptr) __builtin_prefetch(ptr, 0, 2) -#define PREFETCHT2(ptr) __builtin_prefetch(ptr, 0, 1) -#endif - arrow::Result> VeloxSortBasedShuffleWriter::create( uint32_t numPartitions, std::unique_ptr partitionWriter, @@ -154,83 +127,71 @@ arrow::Status VeloxSortBasedShuffleWriter::write(std::shared_ptr return arrow::Status::OK(); } -arrow::Status VeloxSortBasedShuffleWriter::evictBatch(uint32_t partitionId, facebook::velox::RowTypePtr* rowTypePtr) { +arrow::Status VeloxSortBasedShuffleWriter::evictBatch(uint32_t partitionId) { int64_t rawSize = batch_->size(); bufferOutputStream_->seekp(0); batch_->flush(bufferOutputStream_.get()); auto buffer = bufferOutputStream_->getBuffer(); RETURN_NOT_OK(partitionWriter_->evict(partitionId, rawSize, buffer->as(), buffer->size())); batch_ = std::make_unique(veloxPool_.get(), serde_.get()); - batch_->createStreamTree(*rowTypePtr, options_.bufferSize, &serdeOptions_); + batch_->createStreamTree(rowType_, options_.bufferSize, &serdeOptions_); return arrow::Status::OK(); } arrow::Status VeloxSortBasedShuffleWriter::evictRowVector(uint32_t partitionId) { - int32_t rowNum = 0; - const int32_t maxBatchNum = options_.bufferSize; - auto rowTypePtr = std::static_pointer_cast(rowType_.value()); + int32_t accumulatedRows = 0; + const int32_t maxRowsPerBatch = options_.bufferSize; if (options_.partitioning != Partitioning::kSingle) { if (auto it = rowVectorIndexMap_.find(partitionId); it != rowVectorIndexMap_.end()) { - auto rowVectorIndex = it->second; - const int32_t outputSize = rowVectorIndex.size(); + const auto& rowIndices = it->second; + VELOX_DCHECK(!rowIndices.empty()) - std::map> groupedIndices; - std::map groupedSize; + size_t idx = 0; + const auto outputSize = rowIndices.size(); + while (idx < outputSize) { + auto combinedRowIndex = rowIndices[idx]; + auto inputVectorIndex = static_cast(combinedRowIndex >> 32); + auto startRow = static_cast(combinedRowIndex & 0xFFFFFFFFLL); - int32_t tempVectorIndex = -1; - int32_t baseRowIndex = -1; - int32_t tempRowIndex = -1; - int32_t size = 1; - for (int start = 0; start < outputSize; start++) { - const int64_t rowVector = rowVectorIndex[start]; - const int32_t vectorIndex = static_cast(rowVector >> 32); - const int32_t rowIndex = static_cast(rowVector & 0xFFFFFFFFLL); - if (tempVectorIndex == -1) { - tempVectorIndex = vectorIndex; - baseRowIndex = rowIndex; - tempRowIndex = rowIndex; - } else { - if (vectorIndex == tempVectorIndex && rowIndex == tempRowIndex + 1) { - size += 1; - tempRowIndex = rowIndex; + int32_t numRowsInRange = 1; + std::vector groupedIndices; + + while (++idx < outputSize && (rowIndices[idx] >> 32) == inputVectorIndex) { + auto row = static_cast(rowIndices[idx] & 0xFFFFFFFFLL); + if (row == startRow + numRowsInRange) { + numRowsInRange++; } else { - groupedIndices[tempVectorIndex].push_back({baseRowIndex, size}); - groupedSize[tempVectorIndex] += size; - size = 1; - tempVectorIndex = vectorIndex; - baseRowIndex = rowIndex; - tempRowIndex = rowIndex; + groupedIndices.push_back({startRow, numRowsInRange}); + accumulatedRows += numRowsInRange; + startRow = row; + numRowsInRange = 1; } } - } - groupedIndices[tempVectorIndex].push_back({baseRowIndex, size}); - groupedSize[tempVectorIndex] += size; + groupedIndices.push_back({startRow, numRowsInRange}); + batch_->append(batches_[inputVectorIndex], groupedIndices); - for (auto& pair : groupedIndices) { - batch_->append(batches_[pair.first], pair.second); - rowNum += groupedSize[pair.first]; - if (rowNum >= maxBatchNum) { - rowNum = 0; - RETURN_NOT_OK(evictBatch(partitionId, &rowTypePtr)); + accumulatedRows += numRowsInRange; + // Check whether to evict the data after gathering all rows from one input RowVector. + if (accumulatedRows >= maxRowsPerBatch) { + RETURN_NOT_OK(evictBatch(partitionId)); + accumulatedRows = 0; } } - - rowVectorIndex.clear(); rowVectorIndexMap_.erase(partitionId); } } else { for (facebook::velox::RowVectorPtr rowVectorPtr : batches_) { - rowNum += rowVectorPtr->size(); batch_->append(rowVectorPtr); - if (rowNum >= maxBatchNum) { - RETURN_NOT_OK(evictBatch(partitionId, &rowTypePtr)); - rowNum = 0; + accumulatedRows += rowVectorPtr->size(); + if (accumulatedRows >= maxRowsPerBatch) { + RETURN_NOT_OK(evictBatch(partitionId)); + accumulatedRows = 0; } } } - if (rowNum > 0) { - RETURN_NOT_OK(evictBatch(partitionId, &rowTypePtr)); + if (accumulatedRows > 0) { + RETURN_NOT_OK(evictBatch(partitionId)); } return arrow::Status::OK(); } @@ -255,15 +216,12 @@ arrow::Status VeloxSortBasedShuffleWriter::stop() { } arrow::Status VeloxSortBasedShuffleWriter::initFromRowVector(const facebook::velox::RowVector& rv) { - if (!rowType_.has_value()) { - rowType_ = rv.type(); + if (!rowType_) { + rowType_ = facebook::velox::asRowType(rv.type()); serdeOptions_ = { false, facebook::velox::common::stringToCompressionKind(partitionWriter_->options().compressionTypeStr)}; batch_ = std::make_unique(veloxPool_.get(), serde_.get()); - batch_->createStreamTree( - std::static_pointer_cast(rowType_.value()), - options_.bufferSize, - &serdeOptions_); + batch_->createStreamTree(rowType_, options_.bufferSize, &serdeOptions_); } return arrow::Status::OK(); } diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h index 710590184f9b..417d5e926012 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h @@ -66,8 +66,6 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { arrow::Status evictRowVector(uint32_t partitionId) override; - arrow::Status evictBatch(uint32_t partitionId, facebook::velox::RowTypePtr* rowTypePtr); - private: VeloxSortBasedShuffleWriter( uint32_t numPartitions, @@ -85,9 +83,11 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { arrow::Status doSort(facebook::velox::RowVectorPtr rv, int64_t memLimit); + arrow::Status evictBatch(uint32_t partitionId); + void stat() const; - std::optional rowType_; + facebook::velox::RowTypePtr rowType_; std::unique_ptr batch_; std::unique_ptr bufferOutputStream_; From 63c166d62b69956603a4a0bd8991dc4fdf8b01b1 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Fri, 17 May 2024 22:27:51 +0800 Subject: [PATCH 101/402] [VL] Support simulate task spilling in GenericBenchmark (#5795) --- cpp/velox/benchmarks/GenericBenchmark.cc | 10 +++++++++- cpp/velox/benchmarks/common/BenchmarkUtils.cc | 15 +++++++++++++++ cpp/velox/benchmarks/common/BenchmarkUtils.h | 16 ++++++++++++++++ cpp/velox/memory/VeloxMemoryManager.h | 4 ++++ docs/developers/MicroBenchmarks.md | 4 ++++ 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc index 71d3d96b5330..b7a50800e4ea 100644 --- a/cpp/velox/benchmarks/GenericBenchmark.cc +++ b/cpp/velox/benchmarks/GenericBenchmark.cc @@ -64,6 +64,7 @@ DEFINE_string( DEFINE_string(data, "", "Path to input data files in parquet format, used for shuffle read."); DEFINE_string(conf, "", "Path to the configuration file."); DEFINE_string(write_path, "/tmp", "Path to save the output from write tasks."); +DEFINE_int64(memory_limit, std::numeric_limits::max(), "Memory limit used to trigger spill."); struct WriterMetrics { int64_t splitTime; @@ -149,7 +150,11 @@ auto BM_Generic = [](::benchmark::State& state, setCpu(state.thread_index()); } memory::MemoryManager::testingSetInstance({}); - auto memoryManager = getDefaultMemoryManager(); + + auto memoryManager = std::make_unique( + "generic_benchmark", + gluten::defaultMemoryAllocator(), + std::make_unique(FLAGS_memory_limit)); auto runtime = Runtime::create(kVeloxRuntimeKind, conf); auto plan = getPlanFromFile("Plan", planFile); std::vector splits{}; @@ -182,6 +187,9 @@ auto BM_Generic = [](::benchmark::State& state, } auto resultIter = runtime->createResultIterator(memoryManager.get(), "/tmp/test-spill", std::move(inputIters), conf); + if (auto listener = dynamic_cast(memoryManager->getListener())) { + listener->setIterator(resultIter.get()); + } auto veloxPlan = dynamic_cast(runtime)->getVeloxPlan(); if (FLAGS_with_shuffle) { int64_t shuffleWriteTime; diff --git a/cpp/velox/benchmarks/common/BenchmarkUtils.cc b/cpp/velox/benchmarks/common/BenchmarkUtils.cc index efe1fc60af6c..ccec6f3c40b1 100644 --- a/cpp/velox/benchmarks/common/BenchmarkUtils.cc +++ b/cpp/velox/benchmarks/common/BenchmarkUtils.cc @@ -180,3 +180,18 @@ void cleanupShuffleOutput(const std::string& dataFile, const std::vector= limit_) { + LOG(INFO) << fmt::format( + "reach hard limit {} when need {}, current used {}.", + velox::succinctBytes(limit_), + velox::succinctBytes(diff), + velox::succinctBytes(usedBytes_)); + auto neededBytes = usedBytes_ + diff - limit_; + auto spilledBytes = iterator_->spillFixedSize(neededBytes); + LOG(INFO) << fmt::format("spill finish, got {}.", velox::succinctBytes(spilledBytes)); + } else { + usedBytes_ += diff; + } +} diff --git a/cpp/velox/benchmarks/common/BenchmarkUtils.h b/cpp/velox/benchmarks/common/BenchmarkUtils.h index 79f4a53cbaba..ff5e675f74ce 100644 --- a/cpp/velox/benchmarks/common/BenchmarkUtils.h +++ b/cpp/velox/benchmarks/common/BenchmarkUtils.h @@ -102,3 +102,19 @@ arrow::Status setLocalDirsAndDataFileFromEnv(std::string& dataFile, std::vector& localDirs, bool& isFromEnv); void cleanupShuffleOutput(const std::string& dataFile, const std::vector& localDirs, bool isFromEnv); + +class BenchmarkAllocationListener final : public gluten::AllocationListener { + public: + BenchmarkAllocationListener(uint64_t limit) : limit_(limit) {} + + void setIterator(gluten::ResultIterator* iterator) { + iterator_ = iterator; + } + + void allocationChanged(int64_t diff) override; + + private: + uint64_t usedBytes_{0L}; + uint64_t limit_{0L}; + gluten::ResultIterator* iterator_; +}; diff --git a/cpp/velox/memory/VeloxMemoryManager.h b/cpp/velox/memory/VeloxMemoryManager.h index fda153ada8ce..1e8bcd8c8d5e 100644 --- a/cpp/velox/memory/VeloxMemoryManager.h +++ b/cpp/velox/memory/VeloxMemoryManager.h @@ -60,6 +60,10 @@ class VeloxMemoryManager final : public MemoryManager { void hold() override; + AllocationListener* getListener() const { + return listener_.get(); + } + private: bool tryDestructSafe(); diff --git a/docs/developers/MicroBenchmarks.md b/docs/developers/MicroBenchmarks.md index dc0c3b2a019b..7fc2a535dcf1 100644 --- a/docs/developers/MicroBenchmarks.md +++ b/docs/developers/MicroBenchmarks.md @@ -280,6 +280,10 @@ check [Intel® QuickAssist Technology (QAT) support](../get-started/Velox.md#int For IAA support, please check [Intel® In-memory Analytics Accelerator (IAA/IAX) support](../get-started/Velox.md#intel-in-memory-analytics-accelerator-iaaiax-support) +## Simulate task spilling + +You can simulate task spilling by specify memory hard limit from `--memory_limit`. + ## Simulate Spark with multiple processes and threads You can use below command to launch several processes and threads to simulate parallel execution on From c12380d4f6c096dcc608defedd39e4eef6a8c4a8 Mon Sep 17 00:00:00 2001 From: Yuan Date: Sat, 18 May 2024 11:26:50 +0800 Subject: [PATCH 102/402] [VL][CI] disable nightly job on GHA (#5803) Signed-off-by: Yuan Zhou --- .../workflows/{velox_nightly.yml => velox_nightly.yml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{velox_nightly.yml => velox_nightly.yml.disabled} (100%) diff --git a/.github/workflows/velox_nightly.yml b/.github/workflows/velox_nightly.yml.disabled similarity index 100% rename from .github/workflows/velox_nightly.yml rename to .github/workflows/velox_nightly.yml.disabled From be760ee6e2f8346f679af1f43dc94e029c5579a3 Mon Sep 17 00:00:00 2001 From: Rui Mo Date: Sat, 18 May 2024 16:05:49 +0800 Subject: [PATCH 103/402] [VL] Daily Update Velox Version (2024_05_17) (#5781) --- cpp/velox/compute/WholeStageResultIterator.cc | 12 ++++++------ cpp/velox/compute/WholeStageResultIterator.h | 1 + ep/build-velox/src/get_velox.sh | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 06a7a7c391ab..852c7e3cc277 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -72,6 +72,11 @@ WholeStageResultIterator::WholeStageResultIterator( gluten::updateHdfsTokens(veloxCfg_.get()); #endif spillStrategy_ = veloxCfg_->get(kSpillStrategy, kSpillStrategyDefaultValue); + auto spillThreadNum = veloxCfg_->get(kSpillThreadNum, kSpillThreadNumDefaultValue); + if (spillThreadNum > 0) { + spillExecutor_ = std::make_shared(spillThreadNum); + } + getOrderedNodeIds(veloxPlan_, orderedNodeIds_); // Create task instance. @@ -164,18 +169,13 @@ std::shared_ptr WholeStageResultIterator::createNewVeloxQ std::unordered_map> connectorConfigs; connectorConfigs[kHiveConnectorId] = createConnectorConfig(); - auto spillThreadNum = veloxCfg_->get(kSpillThreadNum, kSpillThreadNumDefaultValue); - std::shared_ptr spillExecutor = nullptr; - if (spillThreadNum > 0) { - spillExecutor = std::make_shared(spillThreadNum); - } std::shared_ptr ctx = std::make_shared( nullptr, facebook::velox::core::QueryConfig{getQueryContextConf()}, connectorConfigs, gluten::VeloxBackend::get()->getAsyncDataCache(), memoryManager_->getAggregateMemoryPool(), - std::move(spillExecutor), + spillExecutor_.get(), ""); return ctx; } diff --git a/cpp/velox/compute/WholeStageResultIterator.h b/cpp/velox/compute/WholeStageResultIterator.h index 0ad3877ff203..5e661f40485a 100644 --- a/cpp/velox/compute/WholeStageResultIterator.h +++ b/cpp/velox/compute/WholeStageResultIterator.h @@ -110,6 +110,7 @@ class WholeStageResultIterator : public ColumnarBatchIterator { /// Spill. std::string spillStrategy_; + std::shared_ptr spillExecutor_ = nullptr; /// Metrics std::unique_ptr metrics_{}; diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 17a0b3796f27..33a82ca57d8e 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_16 +VELOX_BRANCH=2024_05_17 VELOX_HOME="" #Set on run gluten on HDFS From 369a98f3957c3a356de91f17db986abdc7b8c694 Mon Sep 17 00:00:00 2001 From: Wei-Ting Chen Date: Mon, 20 May 2024 08:57:08 +0800 Subject: [PATCH 104/402] Add branch protection rule (#5808) --- .asf.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 47ee239613c3..cfe3edf0cf1b 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -31,7 +31,14 @@ github: squash: true merge: false rebase: false - protected_branches: ~ + protected_branches: + main: + required_pull_request_reviews: + dismiss_stale_reviews: true + required_approving_review_count: 1 + required_signatures: true + required_linear_history: true + required_conversation_resolution: true features: issues: true discussions: true From 08f85d6b81999d6f90ef737c5c3a89d4bdb20de8 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 20 May 2024 15:42:29 +0800 Subject: [PATCH 105/402] [CORE] ASF repo config: Set required_signatures to false (#5810) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index cfe3edf0cf1b..ae4827046242 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -36,7 +36,7 @@ github: required_pull_request_reviews: dismiss_stale_reviews: true required_approving_review_count: 1 - required_signatures: true + required_signatures: false required_linear_history: true required_conversation_resolution: true features: From 10182a52d409cb659169cbfe1d7f1869d9205dce Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 20 May 2024 15:46:39 +0800 Subject: [PATCH 106/402] [CORE] Rework planner C2R / R2C code with new transition facilities (#5767) --- .../clickhouse/CHSparkPlanExecApi.scala | 24 +- .../backendsapi/clickhouse/package.scala | 37 +++ ...kHouseTPCDSParquetGraceHashJoinSuite.scala | 2 +- ...kHouseTPCDSParquetSortMergeJoinSuite.scala | 2 +- .../GlutenClickHouseTPCDSParquetSuite.scala | 2 +- .../velox/VeloxSparkPlanExecApi.scala | 50 ++-- .../gluten/backendsapi/velox/package.scala | 43 ++++ .../datasource/v2/ArrowBatchScanExec.scala | 6 + .../python/ColumnarArrowEvalPythonExec.scala | 2 +- .../execution/ArrowFileSourceScanExec.scala | 6 + .../execution/VeloxParquetWriteSuite.scala | 2 +- .../gluten/backendsapi/SparkPlanExecApi.scala | 21 +- .../expression/ExpressionConverter.scala | 19 +- .../gluten/extension/ColumnarOverrides.scala | 3 +- .../apache/gluten/extension/GlutenPlan.scala | 17 +- .../columnar/ColumnarTransitions.scala | 109 -------- .../columnar/ExpandFallbackPolicy.scala | 15 +- .../columnar/MiscColumnarRules.scala | 32 +-- .../enumerated/EnumeratedApplier.scala | 8 +- .../enumerated/PushFilterToScan.scala | 6 + .../columnar/heuristic/HeuristicApplier.scala | 8 +- .../columnar/transition/Convention.scala | 113 +++++++++ .../columnar/transition/ConventionFunc.scala | 115 +++++++++ .../columnar/transition/ConventionReq.scala | 54 ++++ .../columnar/transition/Transition.scala | 186 ++++++++++++++ .../columnar/transition/Transitions.scala | 164 ++++++++++++ .../columnar/transition/package.scala | 58 +++++ .../gluten/planner/cost/GlutenCostModel.scala | 7 +- .../gluten/planner/property/Convention.scala | 26 +- .../org/apache/gluten/utils/PlanUtil.scala | 46 +--- .../ColumnarCollapseTransformStages.scala | 15 +- .../GlutenWriterColumnarRules.scala | 3 +- .../WholeStageTransformerSuite.scala | 3 +- .../columnar/transition/TransitionSuite.scala | 234 ++++++++++++++++++ .../apache/gluten/test}/FallbackUtil.scala | 8 +- .../gluten/columnarbatch/ArrowBatch.scala | 41 +++ gluten-ut/pom.xml | 7 + .../execution/FallbackStrategiesSuite.scala | 2 +- .../benchmarks/ParquetReadBenchmark.scala | 5 +- .../sql/GlutenStringFunctionsSuite.scala | 2 +- .../execution/FallbackStrategiesSuite.scala | 3 +- .../benchmarks/ParquetReadBenchmark.scala | 5 +- .../sql/GlutenStringFunctionsSuite.scala | 2 +- .../execution/FallbackStrategiesSuite.scala | 3 +- .../benchmarks/ParquetReadBenchmark.scala | 4 +- .../sql/GlutenStringFunctionsSuite.scala | 2 +- .../execution/FallbackStrategiesSuite.scala | 3 +- .../benchmarks/ParquetReadBenchmark.scala | 5 +- .../apache/gluten/sql/shims/SparkShims.scala | 2 + .../sql/shims/spark34/Spark34Shims.scala | 6 + .../sql/shims/spark35/Spark35Shims.scala | 8 +- 51 files changed, 1212 insertions(+), 334 deletions(-) create mode 100644 backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/package.scala create mode 100644 backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/package.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarTransitions.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala create mode 100644 gluten-core/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuite.scala rename gluten-core/src/{main/scala/org/apache/gluten/utils => test/scala/org/apache/gluten/test}/FallbackUtil.scala (92%) create mode 100644 gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 465041621a61..cb706d817e71 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -25,6 +25,7 @@ import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.extension.{CountDistinctWithoutExpand, FallbackBroadcastHashJoin, FallbackBroadcastHashJoinPrepQueryStage, RewriteToDateExpresstionRule} import org.apache.gluten.extension.columnar.AddTransformHintRule import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPreOverrides +import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, WindowFunctionNode} import org.apache.gluten.utils.CHJoinValidateUtil @@ -71,6 +72,9 @@ import scala.collection.mutable.ArrayBuffer class CHSparkPlanExecApi extends SparkPlanExecApi { + /** The columnar-batch type this backend is using. */ + override def batchType: Convention.BatchType = CHBatch + /** Transform GetArrayItem to Substrait. */ override def genGetArrayItemExpressionNode( substraitExprName: String, @@ -89,26 +93,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { ConverterUtils.getTypeNode(original.dataType, original.nullable)) } - /** - * Generate ColumnarToRowExecBase. - * - * @param child - * @return - */ - override def genColumnarToRowExec(child: SparkPlan): ColumnarToRowExecBase = { - CHColumnarToRowExec(child) - } - - /** - * Generate RowToColumnarExec. - * - * @param child - * @return - */ - override def genRowToColumnarExec(child: SparkPlan): RowToColumnarExecBase = { - RowToCHNativeColumnarExec(child) - } - override def genProjectExecTransformer( projectList: Seq[NamedExpression], child: SparkPlan): ProjectExecTransformer = { diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/package.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/package.scala new file mode 100644 index 000000000000..8704fac7bcee --- /dev/null +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/package.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.backendsapi + +import org.apache.gluten.extension.columnar.transition.Convention + +import org.apache.spark.sql.execution.{CHColumnarToRowExec, RowToCHNativeColumnarExec, SparkPlan} + +package object clickhouse { + case object CHBatch extends Convention.BatchType { + fromRow( + () => + (plan: SparkPlan) => { + RowToCHNativeColumnarExec(plan) + }) + + toRow( + () => + (plan: SparkPlan) => { + CHColumnarToRowExec(plan) + }) + } +} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala index 0fe04ea2aa3d..0b7ad9a6d8ac 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Not} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSortMergeJoinSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSortMergeJoinSuite.scala index bbedcda18428..b1b9841a3463 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSortMergeJoinSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSortMergeJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala index 27efba6482cb..a63e47888cb9 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Not} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index cf7d38d62bbb..dc6bafdea597 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -21,12 +21,13 @@ import org.apache.gluten.backendsapi.SparkPlanExecApi import org.apache.gluten.datasource.ArrowConvertorRule import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ -import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} -import org.apache.gluten.extension.{ArrowScanReplaceRule, BloomFilterMightContainJointRewriteRule, CollectRewriteRule, FlushableHashAggregateRule, HLLRewriteRule} +import org.apache.gluten.extension._ import org.apache.gluten.extension.columnar.TransformHints +import org.apache.gluten.extension.columnar.transition.Convention +import org.apache.gluten.extension.columnar.transition.ConventionFunc.BatchOverride import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, IfThenNode} import org.apache.gluten.vectorized.{ColumnarBatchSerializer, ColumnarBatchSerializeResult} @@ -50,6 +51,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{BuildSideRelation, HashedRelationBroadcastMode} @@ -73,6 +75,22 @@ import scala.collection.mutable.ListBuffer class VeloxSparkPlanExecApi extends SparkPlanExecApi { + /** The columnar-batch type this backend is using. */ + override def batchType: Convention.BatchType = { + VeloxBatch + } + + /** + * Overrides [[org.apache.gluten.extension.columnar.transition.ConventionFunc]] Gluten is using to + * determine the convention (its row-based processing / columnar-batch processing support) of a + * plan with a user-defined function that accepts a plan then returns batch type it outputs. + */ + override def batchTypeFunc(): BatchOverride = { + case i: InMemoryTableScanExec + if i.relation.cacheBuilder.serializer.isInstanceOf[ColumnarCachedBatchSerializer] => + VeloxBatch + } + /** * Transform GetArrayItem to Substrait. * @@ -275,28 +293,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, children, expr) } - /** - * * Plans. - */ - - /** - * Generate ColumnarToRowExecBase. - * - * @param child - * @return - */ - override def genColumnarToRowExec(child: SparkPlan): ColumnarToRowExecBase = - VeloxColumnarToRowExec(child) - - /** - * Generate RowToColumnarExec. - * - * @param child - * @return - */ - override def genRowToColumnarExec(child: SparkPlan): RowToColumnarExecBase = - RowToVeloxColumnarExec(child) - /** * Generate FilterExecTransformer. * @@ -857,10 +853,4 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { case other => other } } - - override def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = plan match { - case _: ArrowFileSourceScanExec => true - case _: ArrowBatchScanExec => true - case _ => false - } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/package.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/package.scala new file mode 100644 index 000000000000..8ab68b7fe02d --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/package.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.backendsapi + +import org.apache.gluten.columnarbatch.ArrowBatch +import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} +import org.apache.gluten.extension.columnar.transition.{Convention, TransitionDef} + +import org.apache.spark.sql.execution.SparkPlan + +package object velox { + case object VeloxBatch extends Convention.BatchType { + fromRow( + () => + (plan: SparkPlan) => { + RowToVeloxColumnarExec(plan) + }) + + toRow( + () => + (plan: SparkPlan) => { + VeloxColumnarToRowExec(plan) + }) + + // Velox batch is considered one-way compatible with Arrow batch. + // This is practically achieved by utilizing C++ API VeloxColumnarBatch::from at runtime. + fromBatch(ArrowBatch, TransitionDef.empty) + } +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala index 3c1c538207c5..ee0acbf3f461 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/datasource/v2/ArrowBatchScanExec.scala @@ -16,7 +16,9 @@ */ package org.apache.gluten.execution.datasource.v2 +import org.apache.gluten.columnarbatch.ArrowBatch import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.transition.Convention import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow @@ -31,6 +33,10 @@ case class ArrowBatchScanExec(original: BatchScanExec) @transient lazy val batch: Batch = original.batch + override protected def batchType0(): Convention.BatchType = { + ArrowBatch + } + override lazy val readerFactory: PartitionReaderFactory = original.readerFactory override lazy val inputRDD: RDD[InternalRow] = original.inputRDD diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala index d3112c97410d..fd8dfc25b89d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala @@ -293,7 +293,7 @@ case class ColumnarArrowEvalPythonExec( e => if (!e.isInstanceOf[AttributeReference]) { throw new GlutenException( - "ColumnarArrowEvalPythonExec should only has [AttributeReference] inputs.") + "ColumnarArrowEvalPythonExec should only have [AttributeReference] inputs.") } else if (allInputs.exists(_.semanticEquals(e))) { allInputs.indexWhere(_.semanticEquals(e)) } else { diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala index 133bf88b3cbb..e3298d70424a 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ArrowFileSourceScanExec.scala @@ -16,7 +16,9 @@ */ package org.apache.spark.sql.execution +import org.apache.gluten.columnarbatch.ArrowBatch import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.transition.Convention import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow @@ -39,6 +41,10 @@ case class ArrowFileSourceScanExec(original: FileSourceScanExec) override def doCanonicalize(): FileSourceScanExec = original.doCanonicalize() + override protected def batchType0(): Convention.BatchType = { + ArrowBatch + } + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { val numOutputRows = longMetric("numOutputRows") val scanTime = longMetric("scanTime") diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala index c50c7bd75f24..2e4436a59f96 100644 --- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala +++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.execution.VeloxWholeStageTransformerSuite -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.SparkConf diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index c694d580492e..a6228e6715e8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -19,6 +19,7 @@ package org.apache.gluten.backendsapi import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ +import org.apache.gluten.extension.columnar.transition.{Convention, ConventionFunc} import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, WindowFunctionNode} import org.apache.spark.ShuffleDependency @@ -55,21 +56,15 @@ import scala.collection.JavaConverters._ trait SparkPlanExecApi { - /** - * Generate ColumnarToRowExecBase. - * - * @param child - * @return - */ - def genColumnarToRowExec(child: SparkPlan): ColumnarToRowExecBase + /** The columnar-batch type this backend is using. */ + def batchType: Convention.BatchType /** - * Generate RowToColumnarExec. - * - * @param child - * @return + * Overrides [[org.apache.gluten.extension.columnar.transition.ConventionFunc]] Gluten is using to + * determine the convention (its row-based processing / columnar-batch processing support) of a + * plan with a user-defined function that accepts a plan then returns batch type it outputs. */ - def genRowToColumnarExec(child: SparkPlan): RowToColumnarExecBase + def batchTypeFunc(): ConventionFunc.BatchOverride = PartialFunction.empty /** * Generate FilterExecTransformer. @@ -735,6 +730,4 @@ trait SparkPlanExecApi { arrowEvalPythonExec def maybeCollapseTakeOrderedAndProject(plan: SparkPlan): SparkPlan = plan - - def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = false } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 5aacbed055f4..b64a23e860fa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -19,10 +19,10 @@ package org.apache.gluten.expression import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.execution.{ColumnarToRowExecBase, WholeStageTransformer} +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.test.TestStats -import org.apache.gluten.utils.{DecimalArithmeticUtil, PlanUtil} +import org.apache.gluten.utils.DecimalArithmeticUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} @@ -680,20 +680,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { def convertBroadcastExchangeToColumnar( exchange: BroadcastExchangeExec): ColumnarBroadcastExchangeExec = { - val newChild = exchange.child match { - // get WholeStageTransformer directly - case c2r: ColumnarToRowExecBase => c2r.child - // in fallback case - case plan: UnaryExecNode if !PlanUtil.isGlutenColumnarOp(plan) => - plan.child match { - case _: ColumnarToRowExec => - val wholeStageTransformer = exchange.find(_.isInstanceOf[WholeStageTransformer]) - wholeStageTransformer.getOrElse( - BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec(plan)) - case _ => - BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec(plan) - } - } + val newChild = Transitions.toBackendBatchPlan(exchange.child) ColumnarBroadcastExchangeExec(exchange.mode, newChild) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/ColumnarOverrides.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/ColumnarOverrides.scala index 63127727b2f1..067976b63b2c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/ColumnarOverrides.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/ColumnarOverrides.scala @@ -20,6 +20,7 @@ import org.apache.gluten.{GlutenConfig, GlutenSparkExtensionsInjector} import org.apache.gluten.extension.columnar._ import org.apache.gluten.extension.columnar.enumerated.EnumeratedApplier import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.utils.LogLevelUtil import org.apache.spark.broadcast.Broadcast @@ -115,7 +116,7 @@ case class ColumnarOverrideRules(session: SparkSession) override def postColumnarTransitions: Rule[SparkPlan] = plan => { val outputsColumnar = OutputsColumnarTester.inferOutputsColumnar(plan) val unwrapped = OutputsColumnarTester.unwrap(plan) - val vanillaPlan = ColumnarTransitions.insertTransitions(unwrapped, outputsColumnar) + val vanillaPlan = Transitions.insertTransitions(unwrapped, outputsColumnar) val applier: ColumnarRuleApplier = if (GlutenConfig.getConf.enableRas) { new EnumeratedApplier(session) } else { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala index 85901d21dfe3..033e44b8c4d2 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala @@ -20,6 +20,7 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression.TransformerState +import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.plan.PlanBuilder import org.apache.gluten.substrait.rel.RelNode @@ -50,7 +51,7 @@ object ValidationResult { } /** Every Gluten Operator should extend this trait. */ -trait GlutenPlan extends SparkPlan with LogLevelUtil { +trait GlutenPlan extends SparkPlan with Convention.KnownBatchType with LogLevelUtil { private lazy val validationLogLevel = glutenConf.validationLogLevel private lazy val printStackOnValidationFailure = glutenConf.printStackOnValidationFailure @@ -85,6 +86,20 @@ trait GlutenPlan extends SparkPlan with LogLevelUtil { } } + final override def batchType(): Convention.BatchType = { + if (!supportsColumnar) { + throw new UnsupportedOperationException( + s"Node $nodeName doesn't support columnar-batch processing") + } + val batchType = batchType0() + assert(batchType != Convention.BatchType.None) + batchType + } + + protected def batchType0(): Convention.BatchType = { + BackendsApiManager.getSparkPlanExecApiInstance.batchType + } + protected def doValidateInternal(): ValidationResult = ValidationResult.ok protected def doNativeValidation(context: SubstraitContext, node: RelNode): ValidationResult = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarTransitions.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarTransitions.scala deleted file mode 100644 index 5dd266433d4a..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarTransitions.scala +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension.columnar - -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.utils.PlanUtil - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.rules.{PlanChangeLogger, Rule} -import org.apache.spark.sql.execution.{ApplyColumnarRulesAndInsertTransitions, ColumnarToRowExec, ColumnarToRowTransition, RowToColumnarExec, RowToColumnarTransition, SparkPlan} - -/** See rule code from vanilla Spark: [[ApplyColumnarRulesAndInsertTransitions]]. */ -case class InsertTransitions(outputsColumnar: Boolean) extends Rule[SparkPlan] { - private object RemoveRedundantTransitions extends Rule[SparkPlan] { - override def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - case ColumnarToRowExec(RowToColumnarExec(child)) => child - case RowToColumnarExec(ColumnarToRowExec(child)) => child - } - } - - private val rules = List( - ApplyColumnarRulesAndInsertTransitions(List(), outputsColumnar), - RemoveRedundantTransitions) - override def apply(plan: SparkPlan): SparkPlan = rules.foldLeft(plan) { - case (p, r) => r.apply(p) - } -} - -object RemoveTransitions extends Rule[SparkPlan] { - import ColumnarTransitions._ - override def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - case ColumnarToRowLike(child) => child - case RowToColumnarLike(child) => child - } -} - -// This rule will try to add RowToColumnarExecBase and ColumnarToRowExec -// to support vanilla columnar operators. -case class InsertColumnarToColumnarTransitions(session: SparkSession) extends Rule[SparkPlan] { - @transient private val planChangeLogger = new PlanChangeLogger[SparkPlan]() - - private def replaceWithVanillaColumnarToRow(p: SparkPlan): SparkPlan = p.transformUp { - case plan if PlanUtil.isGlutenColumnarOp(plan) => - plan.withNewChildren(plan.children.map { - case child if PlanUtil.isVanillaColumnarOp(child) => - BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec( - ColumnarToRowExec(child)) - case other => other - }) - } - - private def replaceWithVanillaRowToColumnar(p: SparkPlan): SparkPlan = p.transformUp { - case plan if PlanUtil.isVanillaColumnarOp(plan) => - plan.withNewChildren(plan.children.map { - case child if PlanUtil.isGlutenColumnarOp(child) => - RowToColumnarExec( - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(child)) - case other => other - }) - } - - def apply(plan: SparkPlan): SparkPlan = { - val newPlan = replaceWithVanillaRowToColumnar(replaceWithVanillaColumnarToRow(plan)) - planChangeLogger.logRule(ruleName, plan, newPlan) - newPlan - } -} - -object ColumnarTransitions { - def insertTransitions(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = { - InsertTransitions(outputsColumnar).apply(plan) - } - - // Extractor for Spark/Gluten's C2R - object ColumnarToRowLike { - def unapply(plan: SparkPlan): Option[SparkPlan] = { - plan match { - case c2r: ColumnarToRowTransition => - Some(c2r.child) - case _ => None - } - } - } - - // Extractor for Spark/Gluten's R2C - object RowToColumnarLike { - def unapply(plan: SparkPlan): Option[SparkPlan] = { - plan match { - case c2r: RowToColumnarTransition => - Some(c2r.child) - case _ => None - } - } - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala index 471141f49299..6f8d7cde703b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala @@ -19,7 +19,7 @@ package org.apache.gluten.extension.columnar import org.apache.gluten.GlutenConfig import org.apache.gluten.execution.BroadcastHashJoinExecTransformerBase import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPostOverrides +import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike, Transitions} import org.apache.gluten.utils.PlanUtil import org.apache.spark.rdd.RDD @@ -78,7 +78,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP case _: CommandResultExec | _: ExecutedCommandExec => // ignore // we plan exchange to columnar exchange in columnar rules and the exchange does not // support columnar, so the output columnar is always false in AQE postStageCreationRules - case ColumnarToRowExec(s: Exchange) if isAdaptiveContext => + case ColumnarToRowLike(s: Exchange) if isAdaptiveContext => countFallbackInternal(s) case u: UnaryExecNode if !PlanUtil.isGlutenColumnarOp(u) && PlanUtil.isGlutenTableCache(u.child) => @@ -86,15 +86,15 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP // which is a kind of `ColumnarToRowExec`. transitionCost = transitionCost + 1 countFallbackInternal(u.child) - case ColumnarToRowExec(p: GlutenPlan) => + case ColumnarToRowLike(p: GlutenPlan) => logDebug(s"Find a columnar to row for gluten plan:\n$p") transitionCost = transitionCost + 1 countFallbackInternal(p) - case r: RowToColumnarExec => + case RowToColumnarLike(child) => if (!ignoreRowToColumnar) { transitionCost = transitionCost + 1 } - countFallbackInternal(r.child) + countFallbackInternal(child) case leafPlan: LeafExecNode if PlanUtil.isGlutenTableCache(leafPlan) => case leafPlan: LeafExecNode if !PlanUtil.isGlutenColumnarOp(leafPlan) => // Possible fallback for leaf node. @@ -236,9 +236,8 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP } private def fallbackToRowBasedPlan(outputsColumnar: Boolean): SparkPlan = { - val transformPostOverrides = TransformPostOverrides() - val planWithTransitions = ColumnarTransitions.insertTransitions(originalPlan, outputsColumnar) - transformPostOverrides.apply(planWithTransitions) + val planWithTransitions = Transitions.insertTransitions(originalPlan, outputsColumnar) + planWithTransitions } private def countTransitionCostForVanillaSparkPlan(plan: SparkPlan): Int = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala index 08c63000ec73..fab973ffb0ed 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala @@ -16,8 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.extension.columnar.ColumnarTransitions.ColumnarToRowLike +import org.apache.gluten.extension.columnar.transition.ColumnarToRowLike import org.apache.gluten.utils.{LogLevelUtil, PlanUtil} import org.apache.spark.sql.SparkSession @@ -59,35 +58,6 @@ object MiscColumnarRules { } } - // This rule will try to convert the row-to-columnar and columnar-to-row - // into native implementations. - case class TransformPostOverrides() extends Rule[SparkPlan] { - @transient private val planChangeLogger = new PlanChangeLogger[SparkPlan]() - - def replaceWithTransformerPlan(plan: SparkPlan): SparkPlan = plan.transformDown { - case RowToColumnarExec(child) => - logDebug(s"ColumnarPostOverrides RowToColumnarExec(${child.getClass})") - BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec(child) - case c2r @ ColumnarToRowExec(child) - if PlanUtil.outputNativeColumnarData(child) && - !PlanUtil.outputNativeColumnarSparkCompatibleData(child) => - logDebug(s"ColumnarPostOverrides ColumnarToRowExec(${child.getClass})") - val nativeC2r = BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(child) - if (nativeC2r.doValidate().isValid) { - nativeC2r - } else { - c2r - } - } - - // apply for the physical not final plan - def apply(plan: SparkPlan): SparkPlan = { - val newPlan = replaceWithTransformerPlan(plan) - planChangeLogger.logRule(ruleName, plan, newPlan) - newPlan - } - } - // Remove topmost columnar-to-row otherwise AQE throws error. // See: org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec#newQueryStage // diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index 92d64abf39c9..3f8ee870609e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -19,7 +19,8 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ -import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPostOverrides} +import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow} +import org.apache.gluten.extension.columnar.transition.{InsertTransitions, RemoveTransitions} import org.apache.gluten.extension.columnar.util.AdaptiveContext import org.apache.gluten.metrics.GlutenTimeMetric import org.apache.gluten.utils.{LogLevelUtil, PhysicalPlanSelector} @@ -151,10 +152,7 @@ class EnumeratedApplier(session: SparkSession) */ private def postRules(): List[SparkSession => Rule[SparkPlan]] = List( - (_: SparkSession) => TransformPostOverrides(), - (s: SparkSession) => InsertColumnarToColumnarTransitions(s), - (s: SparkSession) => RemoveTopmostColumnarToRow(s, adaptiveContext.isAdaptiveContext()) - ) ::: + (s: SparkSession) => RemoveTopmostColumnarToRow(s, adaptiveContext.isAdaptiveContext())) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarPostRules() ::: List((_: SparkSession) => ColumnarCollapseTransformStages(GlutenConfig.getConf)) ::: SparkRuleUtil.extendedColumnarRules(session, GlutenConfig.getConf.extendedColumnarPostRules) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala index 7306b734a1d3..388668287091 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala @@ -71,11 +71,17 @@ class PushFilterToScan(validator: Validator) extends RasRule[SparkPlan] { private object FilterAndScan { def unapply(node: SparkPlan): Option[(FilterExec, SparkPlan)] = node match { case f @ FilterExec(cond, ColumnarToRowExec(scan)) => + ensureScan(scan) Some(f, scan) case f @ FilterExec(cond, scan) => + ensureScan(scan) Some(f, scan) case _ => None } + + private def ensureScan(node: SparkPlan): Unit = { + assert(node.isInstanceOf[FileSourceScanExec] || node.isInstanceOf[BatchScanExec]) + } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index 0e905ced11db..2b5b18abb27a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -19,8 +19,9 @@ package org.apache.gluten.extension.columnar.heuristic import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ -import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPostOverrides, TransformPreOverrides} +import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPreOverrides} import org.apache.gluten.extension.columnar.rewrite.RewriteSparkPlanRulesManager +import org.apache.gluten.extension.columnar.transition.{InsertTransitions, RemoveTransitions} import org.apache.gluten.extension.columnar.util.AdaptiveContext import org.apache.gluten.metrics.GlutenTimeMetric import org.apache.gluten.utils.{LogLevelUtil, PhysicalPlanSelector} @@ -146,10 +147,7 @@ class HeuristicApplier(session: SparkSession) */ private def postRules(): List[SparkSession => Rule[SparkPlan]] = List( - (_: SparkSession) => TransformPostOverrides(), - (s: SparkSession) => InsertColumnarToColumnarTransitions(s), - (s: SparkSession) => RemoveTopmostColumnarToRow(s, adaptiveContext.isAdaptiveContext()) - ) ::: + (s: SparkSession) => RemoveTopmostColumnarToRow(s, adaptiveContext.isAdaptiveContext())) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarPostRules() ::: List((_: SparkSession) => ColumnarCollapseTransformStages(GlutenConfig.getConf)) ::: SparkRuleUtil.extendedColumnarRules(session, GlutenConfig.getConf.extendedColumnarPostRules) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala new file mode 100644 index 000000000000..2774497d9c22 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.transition + +import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan} + +/** + * Convention of a query plan consists of the row data type and columnar data type it supports to + * output. + */ +sealed trait Convention { + def rowType: Convention.RowType + def batchType: Convention.BatchType +} + +object Convention { + implicit class ConventionOps(val conv: Convention) extends AnyVal { + def isNone: Boolean = { + conv.rowType == RowType.None && conv.batchType == BatchType.None + } + + def &&(other: Convention): Convention = { + def rowType(): RowType = { + if (conv.rowType == other.rowType) { + return conv.rowType + } + RowType.None + } + def batchType(): BatchType = { + if (conv.batchType == other.batchType) { + return conv.batchType + } + BatchType.None + } + Convention.of(rowType(), batchType()) + } + } + + private case class Impl(override val rowType: RowType, override val batchType: BatchType) + extends Convention + + def get(plan: SparkPlan): Convention = { + ConventionFunc.create().conventionOf(plan) + } + + def of(rowType: RowType, batchType: BatchType): Convention = { + Impl(rowType, batchType) + } + + sealed trait RowType + + object RowType { + // None indicates that the plan doesn't support row-based processing. + final case object None extends RowType + final case object VanillaRow extends RowType + } + + trait BatchType { + final def fromRow(transitionDef: TransitionDef): Unit = { + Transition.factory.update().defineFromRowTransition(this, transitionDef) + } + + final def toRow(transitionDef: TransitionDef): Unit = { + Transition.factory.update().defineToRowTransition(this, transitionDef) + } + + final def fromBatch(from: BatchType, transitionDef: TransitionDef): Unit = { + assert(from != this) + Transition.factory.update().defineBatchTransition(from, this, transitionDef) + } + + final def toBatch(to: BatchType, transitionDef: TransitionDef): Unit = { + assert(to != this) + Transition.factory.update().defineBatchTransition(this, to, transitionDef) + } + } + + object BatchType { + // None indicates that the plan doesn't support batch-based processing. + final case object None extends BatchType + final case object VanillaBatch extends BatchType { + fromRow( + () => + (plan: SparkPlan) => { + RowToColumnarExec(plan) + }) + + toRow( + () => + (plan: SparkPlan) => { + ColumnarToRowExec(plan) + }) + } + } + + trait KnownBatchType { + def batchType(): BatchType + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala new file mode 100644 index 000000000000..28bd1d12caf3 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.transition + +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.extension.columnar.transition.Convention.{BatchType, RowType} +import org.apache.gluten.sql.shims.SparkShimLoader + +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} +import org.apache.spark.sql.execution.exchange.ReusedExchangeExec + +/** ConventionFunc is a utility to derive [[Convention]] from a query plan. */ +trait ConventionFunc { + def conventionOf(plan: SparkPlan): Convention +} + +object ConventionFunc { + type BatchOverride = PartialFunction[SparkPlan, BatchType] + + // For testing, to make things work without a backend loaded. + private var ignoreBackend: Boolean = false + + // Visible for testing + def ignoreBackend[T](body: => T): T = synchronized { + assert(!ignoreBackend) + ignoreBackend = true + try { + body + } finally { + ignoreBackend = false + } + } + + def create(): ConventionFunc = { + synchronized { + if (ignoreBackend) { + // For testing + return new BuiltinFunc(PartialFunction.empty) + } + } + val batchOverride = BackendsApiManager.getSparkPlanExecApiInstance.batchTypeFunc() + new BuiltinFunc(batchOverride) + } + + private class BuiltinFunc(o: BatchOverride) extends ConventionFunc { + + override def conventionOf(plan: SparkPlan): Convention = { + val conv = conventionOf0(plan) + conv + } + + private def conventionOf0(plan: SparkPlan): Convention = plan match { + case p if canPropagateConvention(p) => + val childrenConventions = p.children.map(conventionOf0).distinct + if (childrenConventions.size > 1) { + childrenConventions.reduce(_ && _) + } else { + assert(childrenConventions.size == 1) + childrenConventions.head + } + case q: QueryStageExec => conventionOf0(q.plan) + case r: ReusedExchangeExec => conventionOf0(r.child) + case a: AdaptiveSparkPlanExec => + val rowType = rowTypeOf(a) + val batchType = if (a.supportsColumnar) { + // By default, we execute columnar AQE with backend batch output. + // See org.apache.gluten.extension.columnar.transition.InsertTransitions.apply + BackendsApiManager.getSparkPlanExecApiInstance.batchType + } else { + BatchType.None + } + val conv = Convention.of(rowType, batchType) + conv + case other => + val conv = Convention.of(rowTypeOf(other), batchTypeOf(other)) + conv + } + + private def rowTypeOf(plan: SparkPlan): RowType = { + if (!SparkShimLoader.getSparkShims.supportsRowBased(plan)) { + return RowType.None + } + RowType.VanillaRow + } + + private def batchTypeOf(plan: SparkPlan): BatchType = { + if (!plan.supportsColumnar) { + return BatchType.None + } + o.applyOrElse( + plan, + (p: SparkPlan) => + p match { + case g: Convention.KnownBatchType => g.batchType() + case _ => BatchType.VanillaBatch + } + ) + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala new file mode 100644 index 000000000000..aac2084a7a7e --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.transition + +/** + * ConventionReq describes the requirement for [[Convention]]. This is mostly used in determining + * the acceptable conventions for its children of a parent plan node. + */ +sealed trait ConventionReq { + def requiredRowType: ConventionReq.RowType + def requiredBatchType: ConventionReq.BatchType +} + +object ConventionReq { + sealed trait RowType + + object RowType { + final case object Any extends RowType + final case class Is(t: Convention.RowType) extends RowType { + assert(t != Convention.RowType.None) + } + } + + sealed trait BatchType + + object BatchType { + final case object Any extends BatchType + final case class Is(t: Convention.BatchType) extends BatchType { + assert(t != Convention.BatchType.None) + } + } + + private case class Impl( + override val requiredRowType: RowType, + override val requiredBatchType: BatchType + ) extends ConventionReq + + val any: ConventionReq = Impl(RowType.Any, BatchType.Any) + def of(rowType: RowType, batchType: BatchType): ConventionReq = new Impl(rowType, batchType) +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala new file mode 100644 index 000000000000..9b745f94dfdc --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.transition + +import org.apache.gluten.exception.GlutenException + +import org.apache.spark.sql.execution.SparkPlan + +import scala.collection.mutable + +/** + * Transition is a simple function to convert a query plan to interested [[ConventionReq]]. + * + * Transitions can be registered through the utility APIs in + * [[org.apache.gluten.extension.columnar.transition.Convention.BatchType]]'s definition. + */ +trait Transition { + def apply(plan: SparkPlan): SparkPlan +} + +trait TransitionDef { + def create(): Transition +} + +object TransitionDef { + val empty: TransitionDef = () => Transition.empty +} + +object Transition { + val empty: Transition = (plan: SparkPlan) => plan + val factory: Factory = Factory.newBuiltin() + + def notFound(plan: SparkPlan): GlutenException = { + new GlutenException(s"No viable transition found from plan's child to itself: $plan") + } + + def notFound(plan: SparkPlan, required: ConventionReq): GlutenException = { + new GlutenException(s"No viable transition to [$required] found for plan: $plan") + } + + private class ChainedTransition(first: Transition, second: Transition) extends Transition { + override def apply(plan: SparkPlan): SparkPlan = { + second(first(plan)) + } + } + + private def chain(first: Transition, second: Transition): Transition = { + new ChainedTransition(first, second) + } + + trait Factory { + final def findTransition( + from: Convention, + to: ConventionReq, + otherwise: Exception): Transition = { + findTransition(from, to) { + throw otherwise + } + } + + protected def findTransition(from: Convention, to: ConventionReq)( + orElse: => Transition): Transition + private[transition] def update(): MutableFactory + } + + trait MutableFactory extends Factory { + def defineFromRowTransition(to: Convention.BatchType, transitionDef: TransitionDef): Unit + def defineToRowTransition(from: Convention.BatchType, transitionDef: TransitionDef): Unit + def defineBatchTransition( + from: Convention.BatchType, + to: Convention.BatchType, + transitionDef: TransitionDef): Unit + } + + private object Factory { + def newBuiltin(): Factory = { + new BuiltinFactory + } + + private class BuiltinFactory extends MutableFactory { + private val fromRowTransitions: mutable.Map[Convention.BatchType, TransitionDef] = + mutable.Map() + private val toRowTransitions: mutable.Map[Convention.BatchType, TransitionDef] = mutable.Map() + private val batchTransitions + : mutable.Map[(Convention.BatchType, Convention.BatchType), TransitionDef] = + mutable.Map() + + override def defineFromRowTransition( + to: Convention.BatchType, + transitionDef: TransitionDef): Unit = { + assert(!fromRowTransitions.contains(to)) + fromRowTransitions += to -> transitionDef + } + + override def defineToRowTransition( + from: Convention.BatchType, + transitionDef: TransitionDef): Unit = { + assert(!toRowTransitions.contains(from)) + toRowTransitions += from -> transitionDef + } + + override def defineBatchTransition( + from: Convention.BatchType, + to: Convention.BatchType, + transitionDef: TransitionDef): Unit = { + assert(!batchTransitions.contains((from, to))) + batchTransitions += (from, to) -> transitionDef + } + + override def findTransition(from: Convention, to: ConventionReq)( + orElse: => Transition): Transition = { + assert( + !from.isNone, + "#findTransition called with on a plan that doesn't support either row or columnar " + + "output") + val out = (to.requiredRowType, to.requiredBatchType) match { + case (ConventionReq.RowType.Is(toRowType), ConventionReq.BatchType.Is(toBatchType)) => + if (from.rowType == toRowType && from.batchType == toBatchType) { + return Transition.empty + } else { + throw new UnsupportedOperationException( + "Transiting to plan that both have row and columnar-batch output is not yet " + + "supported") + } + case (ConventionReq.RowType.Is(toRowType), ConventionReq.BatchType.Any) => + from.rowType match { + case Convention.RowType.None => + toRowTransitions.get(from.batchType).map(_.create()).getOrElse(orElse) + case fromRowType => + // We have only one single built-in row type. + assert(toRowType == fromRowType) + Transition.empty + } + case (ConventionReq.RowType.Any, ConventionReq.BatchType.Is(toBatchType)) => + from.batchType match { + case Convention.BatchType.None => + fromRowTransitions.get(toBatchType).map(_.create()).getOrElse(orElse) + case fromBatchType => + if (toBatchType == fromBatchType) { + Transition.empty + } else { + // Batch type conversion needed. + // + // We first look up for batch-to-batch transition. If found one, return that + // transition to caller. Otherwise, look for from/to row transitions, then + // return a bridged batch-to-row-to-batch transition. + if (batchTransitions.contains((fromBatchType, toBatchType))) { + // 1. Found batch-to-batch transition. + batchTransitions((fromBatchType, toBatchType)).create() + } else { + // 2. Otherwise, build up batch-to-row-to-batch transition. + val batchToRow = + toRowTransitions.get(fromBatchType).map(_.create()).getOrElse(orElse) + val rowToBatch = + fromRowTransitions.get(toBatchType).map(_.create()).getOrElse(orElse) + chain(batchToRow, rowToBatch) + } + } + } + case (ConventionReq.RowType.Any, ConventionReq.BatchType.Any) => + Transition.empty + case _ => + throw new UnsupportedOperationException( + s"Illegal convention requirement: $ConventionReq") + } + out + } + + override private[transition] def update(): MutableFactory = this + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala new file mode 100644 index 000000000000..e0758cff7423 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.transition + +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.sql.shims.SparkShimLoader + +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{SparkPlan, UnionExec} +import org.apache.spark.sql.execution.command.DataWritingCommandExec + +import scala.annotation.tailrec + +case class InsertTransitions(outputsColumnar: Boolean) extends Rule[SparkPlan] { + import InsertTransitions._ + private val convFunc = ConventionFunc.create() + + override def apply(plan: SparkPlan): SparkPlan = { + // Remove all transitions at first. + val removed = RemoveTransitions.apply(plan) + val filled = fillWithTransitions(removed) + if (!outputsColumnar) { + return Transitions.toRowPlan(filled) + } + Transitions.toBackendBatchPlan(filled) + } + + private def fillWithTransitions(plan: SparkPlan): SparkPlan = plan.transformUp { + case p => applyForNode(p) + } + + private def applyForNode(node: SparkPlan): SparkPlan = { + if (node.children.isEmpty) { + return node + } + val convReq = childrenConvReqOf(node) + val newChildren = node.children.map { + child => + val from = convFunc.conventionOf(child) + if (from.isNone) { + // For example, a union op with row child and columnar child at the same time, + // The plan is actually not executable and we cannot tell about its convention. + child + } else { + val transition = + Transition.factory.findTransition(from, convReq, Transition.notFound(node)) + val newChild = transition.apply(child) + newChild + } + } + node.withNewChildren(newChildren) + } + + private def childrenConvReqOf(node: SparkPlan): ConventionReq = node match { + // TODO: Consider C2C transitions as well when we have some. + case ColumnarToRowLike(_) | RowToColumnarLike(_) => + // C2R / R2C here since they are already removed by + // RemoveTransitions. + // It's current rule's mission to add C2Rs / R2Cs on demand. + throw new IllegalStateException("Unreachable code") + case write: DataWritingCommandExec if SparkShimLoader.getSparkShims.isPlannedV1Write(write) => + // To align with ApplyColumnarRulesAndInsertTransitions#insertTransitions + ConventionReq.any + case u: UnionExec => + // We force vanilla union to output row data to get best compatibility with vanilla Spark. + // As a result it's a common practice to rewrite it with GlutenPlan for offloading. + ConventionReq.of( + ConventionReq.RowType.Is(Convention.RowType.VanillaRow), + ConventionReq.BatchType.Any) + case other => + // In the normal case, children's convention should follow parent node's convention. + // Note, we don't have consider C2R / R2C here since they are already removed by + // RemoveTransitions. + val thisConv = convFunc.conventionOf(other) + thisConv.asReq() + } +} + +object InsertTransitions { + implicit private class ConventionOps(conv: Convention) { + def asReq(): ConventionReq = { + val rowTypeReq = conv.rowType match { + case Convention.RowType.None => ConventionReq.RowType.Any + case r => ConventionReq.RowType.Is(r) + } + + val batchTypeReq = conv.batchType match { + case Convention.BatchType.None => ConventionReq.BatchType.Any + case b => ConventionReq.BatchType.Is(b) + } + ConventionReq.of(rowTypeReq, batchTypeReq) + } + } +} + +object RemoveTransitions extends Rule[SparkPlan] { + override def apply(plan: SparkPlan): SparkPlan = plan.transformDown { case p => removeForNode(p) } + + @tailrec + private[transition] def removeForNode(plan: SparkPlan): SparkPlan = plan match { + // TODO: Consider C2C transitions as well when we have some. + case ColumnarToRowLike(child) => removeForNode(child) + case RowToColumnarLike(child) => removeForNode(child) + case other => other + } +} + +object Transitions { + def insertTransitions(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = { + val out = InsertTransitions(outputsColumnar).apply(plan) + out + } + + def toRowPlan(plan: SparkPlan): SparkPlan = { + val convFunc = ConventionFunc.create() + val req = ConventionReq.of( + ConventionReq.RowType.Is(Convention.RowType.VanillaRow), + ConventionReq.BatchType.Any) + val removed = RemoveTransitions.removeForNode(plan) + val transition = Transition.factory.findTransition( + convFunc.conventionOf(removed), + req, + Transition.notFound(removed, req)) + val out = transition.apply(removed) + out + } + + def toBackendBatchPlan(plan: SparkPlan): SparkPlan = { + val backendBatchType = BackendsApiManager.getSparkPlanExecApiInstance.batchType + val out = toBatchPlan(plan, backendBatchType) + out + } + + def toVanillaBatchPlan(plan: SparkPlan): SparkPlan = { + val out = toBatchPlan(plan, Convention.BatchType.VanillaBatch) + out + } + + private def toBatchPlan(plan: SparkPlan, toBatchType: Convention.BatchType): SparkPlan = { + val convFunc = ConventionFunc.create() + val req = ConventionReq.of(ConventionReq.RowType.Any, ConventionReq.BatchType.Is(toBatchType)) + val removed = RemoveTransitions.removeForNode(plan) + val transition = Transition.factory.findTransition( + convFunc.conventionOf(removed), + req, + Transition.notFound(removed, req)) + val out = transition.apply(removed) + out + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala new file mode 100644 index 000000000000..b0d04c273efe --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/package.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar + +import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.adaptive.AQEShuffleReadExec +import org.apache.spark.sql.execution.debug.DebugExec + +package object transition { + // These 5 plan operators (as of Spark 3.5) are operators that have the + // same convention with their children. + // + // Extend this list in shim layer once Spark has more. + def canPropagateConvention(plan: SparkPlan): Boolean = plan match { + case p: DebugExec => true + case p: UnionExec => true + case p: AQEShuffleReadExec => true + case p: InputAdapter => true + case p: WholeStageCodegenExec => true + case _ => false + } + + // Extractor for Spark/Gluten's C2R + object ColumnarToRowLike { + def unapply(plan: SparkPlan): Option[SparkPlan] = { + plan match { + case c2r: ColumnarToRowTransition => + Some(c2r.child) + case _ => None + } + } + } + + // Extractor for Spark/Gluten's R2C + object RowToColumnarLike { + def unapply(plan: SparkPlan): Option[SparkPlan] = { + plan match { + case c2r: RowToColumnarTransition => + Some(c2r.child) + case _ => None + } + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala index 2920c0a39819..fa69eedb5f23 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala @@ -16,7 +16,8 @@ */ package org.apache.gluten.planner.cost -import org.apache.gluten.extension.columnar.{ColumnarTransitions, OffloadJoin} +import org.apache.gluten.extension.columnar.OffloadJoin +import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike} import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Cost, CostModel} import org.apache.gluten.utils.PlanUtil @@ -63,8 +64,8 @@ object GlutenCostModel { infLongCost case ColumnarToRowExec(child) => 3L case RowToColumnarExec(child) => 3L - case ColumnarTransitions.ColumnarToRowLike(child) => 3L - case ColumnarTransitions.RowToColumnarLike(child) => 3L + case ColumnarToRowLike(child) => 3L + case RowToColumnarLike(child) => 3L case p if PlanUtil.isGlutenColumnarOp(p) => 2L case p if PlanUtil.isVanillaColumnarOp(p) => 3L // Other row ops. Usually a vanilla row op. diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala index 36751f4ca188..5fe96ab79887 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.planner.property -import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.execution.RowToColumnarExecBase import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.ColumnarTransitions +import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike, Transitions} import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Property, PropertyDef} import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} @@ -62,8 +62,8 @@ object ConventionDef extends PropertyDef[SparkPlan, Convention] { case g: GroupLeafExec => g.propertySet.get(ConventionDef) case ColumnarToRowExec(child) => Conventions.ROW_BASED case RowToColumnarExec(child) => Conventions.VANILLA_COLUMNAR - case ColumnarTransitions.ColumnarToRowLike(child) => Conventions.ROW_BASED - case ColumnarTransitions.RowToColumnarLike(child) => Conventions.GLUTEN_COLUMNAR + case ColumnarToRowLike(child) => Conventions.ROW_BASED + case RowToColumnarLike(child) => Conventions.GLUTEN_COLUMNAR case q: QueryStageExec => conventionOf(q.plan) case r: ReusedExchangeExec => conventionOf(r.child) case a: AdaptiveSparkPlanExec => conventionOf(a.executedPlan) @@ -82,8 +82,8 @@ object ConventionDef extends PropertyDef[SparkPlan, Convention] { constraint: Property[SparkPlan], plan: SparkPlan): Seq[Convention] = plan match { case ColumnarToRowExec(child) => Seq(Conventions.VANILLA_COLUMNAR) - case ColumnarTransitions.ColumnarToRowLike(child) => Seq(Conventions.GLUTEN_COLUMNAR) - case ColumnarTransitions.RowToColumnarLike(child) => Seq(Conventions.ROW_BASED) + case ColumnarToRowLike(child) => Seq(Conventions.GLUTEN_COLUMNAR) + case RowToColumnarLike(child) => Seq(Conventions.ROW_BASED) case p if canPropagateConvention(p) => p.children.map(_ => constraint.asInstanceOf[Convention]) case other => @@ -127,22 +127,18 @@ case class ConventionEnforcerRule(reqConv: Convention) extends RasRule[SparkPlan case (Conventions.ROW_BASED, Conventions.VANILLA_COLUMNAR) => List(RowToColumnarExec(node)) case (Conventions.GLUTEN_COLUMNAR, Conventions.ROW_BASED) => - List(BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(node)) + List(Transitions.toRowPlan(node)) case (Conventions.ROW_BASED, Conventions.GLUTEN_COLUMNAR) => - val attempt = BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec(node) - if (attempt.doValidate().isValid) { + val attempt = Transitions.toBackendBatchPlan(node) + if (attempt.asInstanceOf[RowToColumnarExecBase].doValidate().isValid) { List(attempt) } else { List.empty } case (Conventions.VANILLA_COLUMNAR, Conventions.GLUTEN_COLUMNAR) => - List( - BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec( - ColumnarToRowExec(node))) + List(Transitions.toBackendBatchPlan(ColumnarToRowExec(node))) case (Conventions.GLUTEN_COLUMNAR, Conventions.VANILLA_COLUMNAR) => - List( - RowToColumnarExec( - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(node))) + List(RowToColumnarExec(Transitions.toRowPlan(node))) case _ => List.empty } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala index 4c02687a6fa5..fe7eb5566378 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala @@ -17,18 +17,15 @@ package org.apache.gluten.utils import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.transition.Convention import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive._ import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec -import org.apache.spark.sql.execution.exchange._ object PlanUtil { - def isGlutenTableCacheInternal(i: InMemoryTableScanExec): Boolean = { - // `ColumnarCachedBatchSerializer` is at velox module, so use class name here - i.relation.cacheBuilder.serializer.getClass.getSimpleName == "ColumnarCachedBatchSerializer" && - i.supportsColumnar + private def isGlutenTableCacheInternal(i: InMemoryTableScanExec): Boolean = { + Convention.get(i).batchType == BackendsApiManager.getSparkPlanExecApiInstance.batchType } def isGlutenTableCache(plan: SparkPlan): Boolean = { @@ -42,44 +39,11 @@ object PlanUtil { } } - def outputNativeColumnarData(plan: SparkPlan): Boolean = { - plan match { - case a: AQEShuffleReadExec => outputNativeColumnarData(a.child) - case s: QueryStageExec => outputNativeColumnarData(s.plan) - case s: ReusedExchangeExec => outputNativeColumnarData(s.child) - case s: InputAdapter => outputNativeColumnarData(s.child) - case s: WholeStageCodegenExec => outputNativeColumnarData(s.child) - case s: AdaptiveSparkPlanExec => outputNativeColumnarData(s.executedPlan) - case i: InMemoryTableScanExec => PlanUtil.isGlutenTableCache(i) - case _: GlutenPlan => true - case _ => false - } - } - - def outputNativeColumnarSparkCompatibleData(plan: SparkPlan): Boolean = { - BackendsApiManager.getSparkPlanExecApiInstance.outputNativeColumnarSparkCompatibleData(plan) - } - def isVanillaColumnarOp(plan: SparkPlan): Boolean = { - plan match { - case i: InMemoryTableScanExec => - if (PlanUtil.isGlutenTableCache(i)) { - // `InMemoryTableScanExec` do not need extra RowToColumnar or ColumnarToRow - false - } else { - !plan.isInstanceOf[GlutenPlan] && plan.supportsColumnar - } - case a: AQEShuffleReadExec => isVanillaColumnarOp(a.child) - case s: QueryStageExec => isVanillaColumnarOp(s.plan) - case _: RowToColumnarExec => false - case _: InputAdapter => false - case _: WholeStageCodegenExec => false - case r: ReusedExchangeExec => isVanillaColumnarOp(r.child) - case _ => !plan.isInstanceOf[GlutenPlan] && plan.supportsColumnar - } + Convention.get(plan).batchType == Convention.BatchType.VanillaBatch } def isGlutenColumnarOp(plan: SparkPlan): Boolean = { - plan.isInstanceOf[GlutenPlan] + Convention.get(plan).batchType == BackendsApiManager.getSparkPlanExecApiInstance.batchType } } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala index fce07eab4ec3..23746846e9cf 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution._ +import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.rel.RelBuilder @@ -158,16 +159,18 @@ case class ColumnarCollapseTransformStages( } } -case class ColumnarInputAdapter(child: SparkPlan) extends UnaryExecNode { +case class ColumnarInputAdapter(child: SparkPlan) + extends UnaryExecNode + with Convention.KnownBatchType { override def output: Seq[Attribute] = child.output - override def supportsColumnar: Boolean = child.supportsColumnar - override protected def doExecute(): RDD[InternalRow] = - child.execute() + override def supportsColumnar: Boolean = true + override def batchType(): Convention.BatchType = + BackendsApiManager.getSparkPlanExecApiInstance.batchType + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() override protected def doExecuteColumnar(): RDD[ColumnarBatch] = child.executeColumnar() override def outputPartitioning: Partitioning = child.outputPartitioning override def outputOrdering: Seq[SortOrder] = child.outputOrdering - override def vectorTypes: Option[Seq[String]] = child.vectorTypes - override protected[sql] def doExecuteBroadcast[T](): Broadcast[T] = child.executeBroadcast() + override protected[sql] def doExecuteBroadcast[T](): Broadcast[T] = child.doExecuteBroadcast() override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(child = newChild) // Node name's required to be "InputAdapter" to correctly draw UI graph. diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala index f1adb09e285c..859cca842df3 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.ColumnarToRowExecBase import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession @@ -68,7 +69,7 @@ case class FakeRowAdaptor(child: SparkPlan) if (child.supportsColumnar) { child.executeColumnar() } else { - val r2c = BackendsApiManager.getSparkPlanExecApiInstance.genRowToColumnarExec(child) + val r2c = Transitions.toBackendBatchPlan(child) r2c.executeColumnar() } } diff --git a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index bb1867d96f3c..8e8743857139 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -17,7 +17,8 @@ package org.apache.gluten.execution import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.utils.{Arm, FallbackUtil} +import org.apache.gluten.test.FallbackUtil +import org.apache.gluten.utils.Arm import org.apache.spark.SparkConf import org.apache.spark.internal.Logging diff --git a/gluten-core/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuite.scala new file mode 100644 index 000000000000..5c6d692ae22f --- /dev/null +++ b/gluten-core/src/test/scala/org/apache/gluten/extension/columnar/transition/TransitionSuite.scala @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.transition + +import org.apache.gluten.exception.GlutenException +import org.apache.gluten.extension.GlutenPlan + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution._ +import org.apache.spark.sql.test.SharedSparkSession + +class TransitionSuite extends SharedSparkSession { + import TransitionSuite._ + test("Trivial C2R") { + val in = BatchLeaf(TypeA) + val out = ConventionFunc.ignoreBackend { + Transitions.insertTransitions(in, outputsColumnar = false) + } + assert(out == BatchToRow(TypeA, BatchLeaf(TypeA))) + } + + test("Insert C2R") { + val in = RowUnary(BatchLeaf(TypeA)) + val out = ConventionFunc.ignoreBackend { + Transitions.insertTransitions(in, outputsColumnar = false) + } + assert(out == RowUnary(BatchToRow(TypeA, BatchLeaf(TypeA)))) + } + + test("Insert R2C") { + val in = BatchUnary(TypeA, RowLeaf()) + val out = ConventionFunc.ignoreBackend { + Transitions.insertTransitions(in, outputsColumnar = false) + } + assert(out == BatchToRow(TypeA, BatchUnary(TypeA, RowToBatch(TypeA, RowLeaf())))) + } + + test("Insert C2R2C") { + val in = BatchUnary(TypeA, BatchLeaf(TypeB)) + val out = ConventionFunc.ignoreBackend { + Transitions.insertTransitions(in, outputsColumnar = false) + } + assert( + out == BatchToRow( + TypeA, + BatchUnary(TypeA, RowToBatch(TypeA, BatchToRow(TypeB, BatchLeaf(TypeB)))))) + } + + test("Insert C2C") { + val in = BatchUnary(TypeA, BatchLeaf(TypeC)) + val out = ConventionFunc.ignoreBackend { + Transitions.insertTransitions(in, outputsColumnar = false) + } + assert( + out == BatchToRow( + TypeA, + BatchUnary(TypeA, BatchToBatch(from = TypeC, to = TypeA, BatchLeaf(TypeC))))) + } + + test("No transitions found") { + val in = BatchUnary(TypeA, BatchLeaf(TypeD)) + assertThrows[GlutenException] { + ConventionFunc.ignoreBackend { + Transitions.insertTransitions(in, outputsColumnar = false) + } + } + } +} + +object TransitionSuite { + object TypeA extends Convention.BatchType { + fromRow( + () => + (plan: SparkPlan) => { + RowToBatch(this, plan) + }) + + toRow( + () => + (plan: SparkPlan) => { + BatchToRow(this, plan) + }) + } + + object TypeB extends Convention.BatchType { + fromRow( + () => + (plan: SparkPlan) => { + RowToBatch(this, plan) + }) + + toRow( + () => + (plan: SparkPlan) => { + BatchToRow(this, plan) + }) + } + + object TypeC extends Convention.BatchType { + fromRow( + () => + (plan: SparkPlan) => { + RowToBatch(this, plan) + }) + + toRow( + () => + (plan: SparkPlan) => { + BatchToRow(this, plan) + }) + + fromBatch( + TypeA, + () => + (plan: SparkPlan) => { + BatchToBatch(TypeA, this, plan) + }) + + toBatch( + TypeA, + () => + (plan: SparkPlan) => { + BatchToBatch(this, TypeA, plan) + }) + } + + object TypeD extends Convention.BatchType {} + + case class RowToBatch(toBatchType: Convention.BatchType, override val child: SparkPlan) + extends RowToColumnarTransition + with GlutenPlan { + override def supportsColumnar: Boolean = true + override protected def batchType0(): Convention.BatchType = toBatchType + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) + override protected def doExecute(): RDD[InternalRow] = + throw new UnsupportedOperationException() + override def output: Seq[Attribute] = child.output + } + + case class BatchToRow(fromBatchType: Convention.BatchType, override val child: SparkPlan) + extends ColumnarToRowTransition { + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) + override protected def doExecute(): RDD[InternalRow] = + throw new UnsupportedOperationException() + override def output: Seq[Attribute] = child.output + } + + case class BatchToBatch( + from: Convention.BatchType, + to: Convention.BatchType, + override val child: SparkPlan) + extends UnaryExecNode + with GlutenPlan { + override def supportsColumnar: Boolean = true + override protected def batchType0(): Convention.BatchType = to + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = child.output + } + + case class BatchLeaf(override val batchType0: Convention.BatchType) + extends LeafExecNode + with GlutenPlan { + override def supportsColumnar: Boolean = true + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = List.empty + } + + case class BatchUnary( + override val batchType0: Convention.BatchType, + override val child: SparkPlan) + extends UnaryExecNode + with GlutenPlan { + override def supportsColumnar: Boolean = true + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = child.output + } + + case class BatchBinary( + override val batchType0: Convention.BatchType, + override val left: SparkPlan, + override val right: SparkPlan) + extends BinaryExecNode + with GlutenPlan { + override def supportsColumnar: Boolean = true + override protected def withNewChildrenInternal( + newLeft: SparkPlan, + newRight: SparkPlan): SparkPlan = copy(left = newLeft, right = newRight) + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = left.output ++ right.output + } + + case class RowLeaf() extends LeafExecNode { + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = List.empty + } + + case class RowUnary(override val child: SparkPlan) extends UnaryExecNode { + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = child.output + } + + case class RowBinary(override val left: SparkPlan, override val right: SparkPlan) + extends BinaryExecNode { + override protected def withNewChildrenInternal( + newLeft: SparkPlan, + newRight: SparkPlan): SparkPlan = copy(left = newLeft, right = newRight) + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + override def output: Seq[Attribute] = left.output ++ right.output + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/FallbackUtil.scala b/gluten-core/src/test/scala/org/apache/gluten/test/FallbackUtil.scala similarity index 92% rename from gluten-core/src/main/scala/org/apache/gluten/utils/FallbackUtil.scala rename to gluten-core/src/test/scala/org/apache/gluten/test/FallbackUtil.scala index c40cdd675fa7..d2626ab275ce 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/FallbackUtil.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/test/FallbackUtil.scala @@ -14,7 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.utils +package org.apache.gluten.test + +import org.apache.gluten.extension.GlutenPlan import org.apache.spark.internal.Logging import org.apache.spark.sql.execution._ @@ -66,12 +68,12 @@ object FallbackUtil extends Logging with AdaptiveSparkPlanHelper { var fallbackOperator: Seq[SparkPlan] = null if (plan.isInstanceOf[AdaptiveSparkPlanExec]) { fallbackOperator = collectWithSubqueries(plan) { - case plan if !PlanUtil.isGlutenColumnarOp(plan) && !skip(plan) => + case plan if !plan.isInstanceOf[GlutenPlan] && !skip(plan) => plan } } else { fallbackOperator = plan.collectWithSubqueries { - case plan if !PlanUtil.isGlutenColumnarOp(plan) && !skip(plan) => + case plan if !plan.isInstanceOf[GlutenPlan] && !skip(plan) => plan } } diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala new file mode 100644 index 000000000000..3f40793d9da5 --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gluten.columnarbatch + +import org.apache.gluten.extension.columnar.transition.Convention + +import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan} + +/** + * ArrowBatch stands for Gluten's Arrow-based columnar batch implementation. Vanilla Spark's + * ColumnarBatch consisting of [[org.apache.spark.sql.vectorized.ArrowColumnVector]]s is still + * treated as [[Convention.BatchType.VanillaBatch]]. + * + * As of now, ArrowBatch should have [[org.apache.gluten.vectorized.ArrowWritableColumnVector]]s + * populated in it. ArrowBatch can be loaded from / offloaded to native to C++ ArrowColumnarBatch + * through API in [[ColumnarBatches]]. After being offloaded, ArrowBatch is no longer considered a + * legal ArrowBatch and cannot be accepted by trivial ColumnarToRowExec. To follow that rule, Any + * plan with this batch type should promise it emits loaded batch only. + */ +object ArrowBatch extends Convention.BatchType { + toRow( + () => + (plan: SparkPlan) => { + ColumnarToRowExec(plan) + }) +} diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml index 8015b5cecaa8..2396087fcc33 100644 --- a/gluten-ut/pom.xml +++ b/gluten-ut/pom.xml @@ -42,6 +42,13 @@ ${project.version} compile + + org.apache.gluten + gluten-core + ${project.version} + test-jar + test + org.apache.spark spark-core_${scala.binary.version} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index e2a99ef7e438..7c7aa08791e8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -18,8 +18,8 @@ package org.apache.spark.sql.execution import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.InsertTransitions import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier +import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector import org.apache.spark.rdd.RDD diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 89a0ee18a0a0..ad08318bbde5 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.gluten.vectorized.JniLibLoader @@ -124,8 +124,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { val newWholeStage = wholeStageTransform.withNewChildren(Seq(fileScan)) // generate ColumnarToRow - val columnarToRowPlan = - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(newWholeStage) + val columnarToRowPlan = Transitions.toRowPlan(newWholeStage) val newWholeStageRDD = newWholeStage.executeColumnar() val newColumnarToRowRDD = columnarToRowPlan.execute() diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index c5e4fa0a4250..c58284e4403b 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper import org.apache.spark.sql.functions._ diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 31517a141c9d..fff883d49e86 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -19,8 +19,9 @@ package org.apache.spark.sql.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, InsertTransitions, TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, TRANSFORM_UNSUPPORTED, TransformHints} import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier +import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector import org.apache.spark.rdd.RDD diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index d99d529241ec..7d8a292042e8 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.gluten.vectorized.JniLibLoader @@ -124,8 +124,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { val newWholeStage = wholeStageTransform.withNewChildren(Seq(fileScan)) // generate ColumnarToRow - val columnarToRowPlan = - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(newWholeStage) + val columnarToRowPlan = Transitions.toRowPlan(newWholeStage) val newWholeStageRDD = newWholeStage.executeColumnar() val newColumnarToRowRDD = columnarToRowPlan.execute() diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index c5e4fa0a4250..c58284e4403b 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper import org.apache.spark.sql.functions._ diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 079620bf8166..7976288dd4ef 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -19,8 +19,9 @@ package org.apache.spark.sql.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, InsertTransitions, TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, TRANSFORM_UNSUPPORTED, TransformHints} import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier +import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector import org.apache.spark.rdd.RDD diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index d99d529241ec..b5481f4d88c4 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.gluten.vectorized.JniLibLoader @@ -125,7 +125,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { // generate ColumnarToRow val columnarToRowPlan = - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(newWholeStage) + Transitions.toBackendBatchPlan(newWholeStage) val newWholeStageRDD = newWholeStage.executeColumnar() val newColumnarToRowRDD = columnarToRowPlan.execute() diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index c5e4fa0a4250..c58284e4403b 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.utils.FallbackUtil +import org.apache.gluten.test.FallbackUtil import org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper import org.apache.spark.sql.functions._ diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 079620bf8166..7976288dd4ef 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -19,8 +19,9 @@ package org.apache.spark.sql.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, InsertTransitions, TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, TRANSFORM_UNSUPPORTED, TransformHints} import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier +import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector import org.apache.spark.rdd.RDD diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index d99d529241ec..7d8a292042e8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} +import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.gluten.vectorized.JniLibLoader @@ -124,8 +124,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { val newWholeStage = wholeStageTransform.withNewChildren(Seq(fileScan)) // generate ColumnarToRow - val columnarToRowPlan = - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarToRowExec(newWholeStage) + val columnarToRowPlan = Transitions.toRowPlan(newWholeStage) val newWholeStageRDD = newWholeStage.executeColumnar() val newColumnarToRowRDD = columnarToRowPlan.execute() diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala index dbefc22ef7a2..fd8cd24c393e 100644 --- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala @@ -39,6 +39,7 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{InputPartition, Scan} import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, GlobalLimitExec, SparkPlan, TakeOrderedAndProjectExec} import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec +import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionDirectory, PartitionedFile, PartitioningAwareFileIndex, WriteJobDescription, WriteTaskResult} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan @@ -232,4 +233,5 @@ trait SparkShims { def dateTimestampFormatInReadIsDefaultValue(csvOptions: CSVOptions, timeZone: String): Boolean + def isPlannedV1Write(write: DataWritingCommandExec): Boolean = false } diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 0b045972a8b9..6d70d67f313c 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -40,11 +40,13 @@ import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} @@ -420,4 +422,8 @@ class Spark34Shims extends SparkShims { csvOptions.timestampFormatInRead == default.timestampFormatInRead && csvOptions.timestampNTZFormatInRead == default.timestampNTZFormatInRead } + + override def isPlannedV1Write(write: DataWritingCommandExec): Boolean = { + write.cmd.isInstanceOf[V1WriteCommand] && SQLConf.get.plannedWriteEnabled + } } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index c839c8c2af03..00f9d62fd211 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, InternalRow} import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.aggregate.{BloomFilterAggregate, RegrIntercept, RegrR2, RegrReplacement, RegrSlope, RegrSXY, TypedImperativeAggregate} +import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution, KeyGroupedPartitioning, Partitioning} import org.apache.spark.sql.catalyst.rules.Rule @@ -40,12 +40,14 @@ import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ShuffleExchangeLike} import org.apache.spark.sql.execution.window.{WindowGroupLimitExec, WindowGroupLimitExecShim} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} @@ -448,4 +450,8 @@ class Spark35Shims extends SparkShims { csvOptions.timestampFormatInRead == default.timestampFormatInRead && csvOptions.timestampNTZFormatInRead == default.timestampNTZFormatInRead } + + override def isPlannedV1Write(write: DataWritingCommandExec): Boolean = { + write.cmd.isInstanceOf[V1WriteCommand] && SQLConf.get.plannedWriteEnabled + } } From 73e11a3161c352282783ca2af8503f032d61d5a3 Mon Sep 17 00:00:00 2001 From: exmy Date: Mon, 20 May 2024 16:10:33 +0800 Subject: [PATCH 107/402] [GLUTEN-5741][CH] Fix core dump when executor exits (#5787) --- cpp-ch/local-engine/local_engine_jni.cpp | 1 + .../gluten/vectorized/JniLibLoader.java | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 63341cc53eab..c7721b470cf8 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -201,6 +201,7 @@ JNIEXPORT jint JNI_OnLoad(JavaVM * vm, void * /*reserved*/) JNIEXPORT void JNI_OnUnload(JavaVM * vm, void * /*reserved*/) { + LOG_INFO(&Poco::Logger::get("jni"), "start jni onUnload"); local_engine::BackendFinalizerUtil::finalizeGlobally(); JNIEnv * env; diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index 31e6b0493d0b..b1feb1e5baeb 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -82,7 +82,21 @@ public static synchronized void forceUnloadAll() { loaded.forEach(JniLibLoader::unloadFromPath); } + private static String toRealPath(String libPath) { + String realPath = libPath; + try { + while (Files.isSymbolicLink(Paths.get(realPath))) { + realPath = Files.readSymbolicLink(Paths.get(realPath)).toString(); + } + LOG.info("Read real path {} for libPath {}", realPath, libPath); + return realPath; + } catch (Throwable th) { + throw new GlutenException("Error to read real path for libPath: " + libPath, th); + } + } + private static synchronized void loadFromPath0(String libPath, boolean requireUnload) { + libPath = toRealPath(libPath); if (LOADED_LIBRARY_PATHS.contains(libPath)) { LOG.debug("Library in path {} has already been loaded, skipping", libPath); } else { @@ -125,13 +139,10 @@ public static synchronized void unloadFromPath(String libPath) { return; } + LOG.info("Starting unload library path: {} ", libPath); REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath); try { - while (Files.isSymbolicLink(Paths.get(libPath))) { - libPath = Files.readSymbolicLink(Paths.get(libPath)).toString(); - } - ClassLoader classLoader = JniLibLoader.class.getClassLoader(); Field field = ClassLoader.class.getDeclaredField("nativeLibraries"); field.setAccessible(true); @@ -151,6 +162,7 @@ public static synchronized void unloadFromPath(String libPath) { String libFileName = libFile.getName(); if (verboseFileName.equals(libFileName)) { + LOG.info("Finalizing library file: {}", libFileName); Method finalize = object.getClass().getDeclaredMethod("finalize"); finalize.setAccessible(true); finalize.invoke(object); From 34282cf0453edc3368b4914179ce8cd4dacf4c08 Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Mon, 20 May 2024 16:19:36 +0800 Subject: [PATCH 108/402] [CORE] Unify the aggregate function name mapping (#5809) --- .../CHHashAggregateExecTransformer.scala | 2 +- .../velox/VeloxSparkPlanExecApi.scala | 2 +- .../HashAggregateExecTransformer.scala | 19 +------- .../spark/sql/expression/UDFResolver.scala | 1 + .../AggregateFunctionsBuilder.scala | 44 +++++++++---------- .../gluten/expression/ExpressionNames.scala | 1 + 6 files changed, 27 insertions(+), 42 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala index d4f2f9eb3874..4a4d345db1dd 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala @@ -294,7 +294,7 @@ case class CHHashAggregateExecTransformer( ._1 .get } else { - AggregateFunctionsBuilder.getSubstraitFunctionName(aggregateFunc).get + AggregateFunctionsBuilder.getSubstraitFunctionName(aggregateFunc) } ConverterUtils.genColumnNameWithExprId(resultAttr) + "#Partial#" + aggFunctionName } diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index dc6bafdea597..69e56b422561 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -786,7 +786,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { Seq( Sig[HLLAdapter](ExpressionNames.APPROX_DISTINCT), Sig[UDFExpression](ExpressionNames.UDF_PLACEHOLDER), - Sig[UserDefinedAggregateFunction](ExpressionNames.UDF_PLACEHOLDER), + Sig[UserDefinedAggregateFunction](ExpressionNames.UDAF_PLACEHOLDER), Sig[NaNvl](ExpressionNames.NANVL), Sig[VeloxCollectList](ExpressionNames.COLLECT_LIST), Sig[VeloxCollectSet](ExpressionNames.COLLECT_SET), diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala index 26d30606ddb7..2f447572406b 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala @@ -31,7 +31,6 @@ import org.apache.gluten.utils.VeloxIntermediateData import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.execution._ -import org.apache.spark.sql.expression.UserDefinedAggregateFunction import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -706,27 +705,13 @@ object VeloxAggregateFunctionsBuilder { aggregateFunc: AggregateFunction, mode: AggregateMode): Long = { val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - - var sigName = ExpressionMappings.expressionsMap.get(aggregateFunc.getClass) - if (sigName.isEmpty) { - throw new GlutenNotSupportException(s"not currently supported: $aggregateFunc.") - } - - aggregateFunc match { - case First(_, ignoreNulls) => - if (ignoreNulls) sigName = Some(ExpressionNames.FIRST_IGNORE_NULL) - case Last(_, ignoreNulls) => - if (ignoreNulls) sigName = Some(ExpressionNames.LAST_IGNORE_NULL) - case UserDefinedAggregateFunction(name, _, _, _, _) => - sigName = Some(name) - case _ => - } + val sigName = AggregateFunctionsBuilder.getSubstraitFunctionName(aggregateFunc) ExpressionBuilder.newScalarFunction( functionMap, ConverterUtils.makeFuncName( // Substrait-to-Velox procedure will choose appropriate companion function if needed. - sigName.get, + sigName, VeloxIntermediateData.getInputTypes(aggregateFunc, mode == PartialMerge || mode == Final), FunctionConfig.REQ ) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index 847e5a2e683e..c34c1ae7f121 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -53,6 +53,7 @@ case class UserDefinedAggregateFunction( children: Seq[Expression], override val aggBufferAttributes: Seq[AttributeReference]) extends AggregateFunction { + override def prettyName: String = name override def aggBufferSchema: StructType = StructType( diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/AggregateFunctionsBuilder.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/AggregateFunctionsBuilder.scala index f83ab1262b17..6ac2c67eb086 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/AggregateFunctionsBuilder.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/AggregateFunctionsBuilder.scala @@ -34,15 +34,18 @@ object AggregateFunctionsBuilder { ExpressionMappings.expressionExtensionTransformer.extensionExpressionsMapping.contains( aggregateFunc.getClass) ) { - ExpressionMappings.expressionExtensionTransformer.buildCustomAggregateFunction( - aggregateFunc) + val (substraitAggFuncName, inputTypes) = + ExpressionMappings.expressionExtensionTransformer.buildCustomAggregateFunction( + aggregateFunc) + assert(substraitAggFuncName.isDefined) + (substraitAggFuncName.get, inputTypes) } else { val substraitAggFuncName = getSubstraitFunctionName(aggregateFunc) // Check whether each backend supports this aggregate function. if ( !BackendsApiManager.getValidatorApiInstance.doExprValidate( - substraitAggFuncName.get, + substraitAggFuncName, aggregateFunc) ) { throw new GlutenNotSupportException( @@ -55,30 +58,25 @@ object AggregateFunctionsBuilder { ExpressionBuilder.newScalarFunction( functionMap, - ConverterUtils.makeFuncName(substraitAggFuncName.get, inputTypes, FunctionConfig.REQ)) + ConverterUtils.makeFuncName(substraitAggFuncName, inputTypes, FunctionConfig.REQ)) } - def getSubstraitFunctionName(aggregateFunc: AggregateFunction): Option[String] = { - val substraitAggFuncName = aggregateFunc match { - case first @ First(_, ignoreNull) => - if (ignoreNull) { - Some(ExpressionNames.FIRST_IGNORE_NULL) - } else { - Some(ExpressionNames.FIRST) + def getSubstraitFunctionName(aggregateFunc: AggregateFunction): String = { + aggregateFunc match { + case First(_, ignoreNulls) if ignoreNulls => + ExpressionNames.FIRST_IGNORE_NULL + case Last(_, ignoreNulls) if ignoreNulls => + ExpressionNames.LAST_IGNORE_NULL + case _ => + val nameOpt = ExpressionMappings.expressionsMap.get(aggregateFunc.getClass) + if (nameOpt.isEmpty) { + throw new UnsupportedOperationException( + s"Could not find a valid substrait mapping name for $aggregateFunc.") } - case last @ Last(_, ignoreNulls) => - if (ignoreNulls) { - Some(ExpressionNames.LAST_IGNORE_NULL) - } else { - Some(ExpressionNames.LAST) + nameOpt.get match { + case ExpressionNames.UDAF_PLACEHOLDER => aggregateFunc.prettyName + case name => name } - case _ => - ExpressionMappings.expressionsMap.get(aggregateFunc.getClass) - } - if (substraitAggFuncName.isEmpty) { - throw new UnsupportedOperationException( - s"Could not find valid a substrait mapping name for $aggregateFunc.") } - substraitAggFuncName } } diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index b9e247a8d439..735bb7a23f41 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -323,4 +323,5 @@ object ExpressionNames { // A placeholder for native UDF functions final val UDF_PLACEHOLDER = "udf_placeholder" + final val UDAF_PLACEHOLDER = "udaf_placeholder" } From 516fda140400c94f6e318c514ce75b675240dbda Mon Sep 17 00:00:00 2001 From: James Xu Date: Mon, 20 May 2024 16:55:24 +0800 Subject: [PATCH 109/402] [GLUTEN-5792][CORE] Fix build on macOS (#5800) --- cpp/CMakeLists.txt | 2 ++ cpp/core/CMakeLists.txt | 1 - cpp/core/shuffle/HashPartitioner.cc | 2 +- cpp/core/shuffle/Partitioner.h | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 28c28a5bd000..d6e3eeb133f7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -118,6 +118,8 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") "-Wno-implicit-int-float-conversion \ -Wno-nullability-completeness \ -Wno-mismatched-tags \ + -Wno-error=unused-private-field \ + -Wno-error=pessimizing-move \ ${KNOWN_WARNINGS}") else() message(FATAL_ERROR "Unsupported compiler ID: ${CMAKE_CXX_COMPILER_ID}") diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 4a8ae0e47c4b..e2d312abaacb 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -30,7 +30,6 @@ endif() set(BOOST_MIN_VERSION "1.42.0") find_package(Boost REQUIRED) INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}) - set(source_root_directory ${CMAKE_CURRENT_SOURCE_DIR}) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") diff --git a/cpp/core/shuffle/HashPartitioner.cc b/cpp/core/shuffle/HashPartitioner.cc index 4a26dc67b2cb..6551307d2d5d 100644 --- a/cpp/core/shuffle/HashPartitioner.cc +++ b/cpp/core/shuffle/HashPartitioner.cc @@ -31,7 +31,7 @@ int32_t computePid(const int32_t* pidArr, int64_t i, int32_t numPartitions) { : [num_partitions] "r"(numPartitions), [tmp] "r"(0)); #else if (pid < 0) { - pid += numPartitions_; + pid += numPartitions; } #endif return pid; diff --git a/cpp/core/shuffle/Partitioner.h b/cpp/core/shuffle/Partitioner.h index b233f5b82673..cc44a2bee3e2 100644 --- a/cpp/core/shuffle/Partitioner.h +++ b/cpp/core/shuffle/Partitioner.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include From 4a951ba78f07926cb1a85429fa43584eba38be44 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 20 May 2024 19:23:38 +0800 Subject: [PATCH 110/402] [VL] Move memory reservation block computation logic into AllocationListener --- cpp/core/config/GlutenConfig.h | 3 + cpp/core/jni/JniCommon.h | 59 +++----- cpp/core/jni/JniWrapper.cc | 5 +- cpp/core/memory/AllocationListener.h | 43 ++++++ cpp/core/memory/MemoryAllocator.cc | 55 +++----- cpp/core/memory/MemoryAllocator.h | 5 +- cpp/velox/benchmarks/common/BenchmarkUtils.cc | 3 +- cpp/velox/memory/VeloxMemoryManager.cc | 17 ++- cpp/velox/memory/VeloxMemoryManager.h | 8 +- cpp/velox/tests/CMakeLists.txt | 14 +- cpp/velox/tests/FunctionTest.cc | 19 ++- cpp/velox/tests/MemoryManagerTest.cc | 129 ++++++++++++++++++ .../Substrait2VeloxPlanConversionTest.cc | 3 +- .../tests/Substrait2VeloxPlanValidatorTest.cc | 3 - .../tests/VeloxColumnarBatchSerializerTest.cc | 10 +- cpp/velox/tests/VeloxColumnarBatchTest.cc | 7 +- cpp/velox/tests/VeloxColumnarToRowTest.cc | 6 +- cpp/velox/tests/VeloxRowToColumnarTest.cc | 3 - .../memory/nmm/NativeMemoryManager.java | 13 +- 19 files changed, 261 insertions(+), 144 deletions(-) create mode 100644 cpp/velox/tests/MemoryManagerTest.cc diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index 16a18f6be903..a039537b78ba 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -42,6 +42,9 @@ const std::string kSparkOffHeapMemory = "spark.gluten.memory.offHeap.size.in.byt const std::string kSparkTaskOffHeapMemory = "spark.gluten.memory.task.offHeap.size.in.bytes"; +const std::string kMemoryReservationBlockSize = "spark.gluten.memory.reservationBlockSize"; +const uint64_t kMemoryReservationBlockSizeDefault = 8 << 20; + const std::string kSparkBatchSize = "spark.gluten.sql.columnar.maxBatchSize"; const std::string kParquetBlockSize = "parquet.block.size"; diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index aa3b2b8840c6..5858a70e9a77 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -322,13 +322,8 @@ static inline gluten::CompressionMode getCompressionMode(JNIEnv* env, jstring co class SparkAllocationListener final : public gluten::AllocationListener { public: - SparkAllocationListener( - JavaVM* vm, - jobject jListenerLocalRef, - jmethodID jReserveMethod, - jmethodID jUnreserveMethod, - int64_t blockSize) - : vm_(vm), jReserveMethod_(jReserveMethod), jUnreserveMethod_(jUnreserveMethod), blockSize_(blockSize) { + SparkAllocationListener(JavaVM* vm, jobject jListenerLocalRef, jmethodID jReserveMethod, jmethodID jUnreserveMethod) + : vm_(vm), jReserveMethod_(jReserveMethod), jUnreserveMethod_(jUnreserveMethod) { JNIEnv* env; attachCurrentThreadAsDaemonOrThrow(vm_, &env); jListenerGlobalRef_ = env->NewGlobalRef(jListenerLocalRef); @@ -350,7 +345,20 @@ class SparkAllocationListener final : public gluten::AllocationListener { } void allocationChanged(int64_t size) override { - updateReservation(size); + if (size == 0) { + return; + } + JNIEnv* env; + attachCurrentThreadAsDaemonOrThrow(vm_, &env); + if (size < 0) { + env->CallLongMethod(jListenerGlobalRef_, jUnreserveMethod_, -size); + checkException(env); + } else { + env->CallLongMethod(jListenerGlobalRef_, jReserveMethod_, size); + checkException(env); + } + bytesReserved_ += size; + maxBytesReserved_ = std::max(bytesReserved_, maxBytesReserved_); } int64_t currentBytes() override { @@ -362,47 +370,12 @@ class SparkAllocationListener final : public gluten::AllocationListener { } private: - int64_t reserve(int64_t diff) { - std::lock_guard lock(mutex_); - bytesReserved_ += diff; - int64_t newBlockCount; - if (bytesReserved_ == 0) { - newBlockCount = 0; - } else { - // ceil to get the required block number - newBlockCount = (bytesReserved_ - 1) / blockSize_ + 1; - } - int64_t bytesGranted = (newBlockCount - blocksReserved_) * blockSize_; - blocksReserved_ = newBlockCount; - maxBytesReserved_ = std::max(maxBytesReserved_, bytesReserved_); - return bytesGranted; - } - - void updateReservation(int64_t diff) { - int64_t granted = reserve(diff); - if (granted == 0) { - return; - } - JNIEnv* env; - attachCurrentThreadAsDaemonOrThrow(vm_, &env); - if (granted < 0) { - env->CallLongMethod(jListenerGlobalRef_, jUnreserveMethod_, -granted); - checkException(env); - } else { - env->CallLongMethod(jListenerGlobalRef_, jReserveMethod_, granted); - checkException(env); - } - } - JavaVM* vm_; jobject jListenerGlobalRef_; jmethodID jReserveMethod_; jmethodID jUnreserveMethod_; - int64_t blockSize_; - int64_t blocksReserved_ = 0L; int64_t bytesReserved_ = 0L; int64_t maxBytesReserved_ = 0L; - std::mutex mutex_; }; class BacktraceAllocationListener final : public gluten::AllocationListener { diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index e70a017e07d0..6a1926317071 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -1309,7 +1309,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_cr jstring jbackendType, jstring jnmmName, jlong allocatorId, - jlong reservationBlockSize, jobject jlistener) { JNI_METHOD_START JavaVM* vm; @@ -1321,8 +1320,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_cr throw gluten::GlutenException("Allocator does not exist or has been closed"); } - std::unique_ptr listener = std::make_unique( - vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod, reservationBlockSize); + std::unique_ptr listener = + std::make_unique(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod); if (gluten::backtrace_allocation) { listener = std::make_unique(std::move(listener)); diff --git a/cpp/core/memory/AllocationListener.h b/cpp/core/memory/AllocationListener.h index 23015e1a04ee..a3c0a72cbc30 100644 --- a/cpp/core/memory/AllocationListener.h +++ b/cpp/core/memory/AllocationListener.h @@ -17,6 +17,7 @@ #pragma once +#include #include namespace gluten { @@ -44,4 +45,46 @@ class AllocationListener { AllocationListener() = default; }; +/// Memory changes will be round to specified block size which aim to decrease delegated listener calls. +class BlockAllocationListener final : public AllocationListener { + public: + BlockAllocationListener(AllocationListener* delegated, uint64_t blockSize) + : delegated_(delegated), blockSize_(blockSize) {} + + void allocationChanged(int64_t diff) override { + if (diff == 0) { + return; + } + if (diff > 0) { + if (reservationBytes_ - usedBytes_ < diff) { + auto roundSize = (diff + (blockSize_ - 1)) / blockSize_ * blockSize_; + delegated_->allocationChanged(roundSize); + reservationBytes_ += roundSize; + peakBytes_ = std::max(peakBytes_, reservationBytes_); + } + usedBytes_ += diff; + } else { + usedBytes_ += diff; + auto unreservedSize = (reservationBytes_ - usedBytes_) / blockSize_ * blockSize_; + delegated_->allocationChanged(-unreservedSize); + reservationBytes_ -= unreservedSize; + } + } + + int64_t currentBytes() override { + return reservationBytes_; + } + + int64_t peakBytes() override { + return peakBytes_; + } + + private: + AllocationListener* delegated_; + uint64_t blockSize_{0L}; + uint64_t usedBytes_{0L}; + uint64_t peakBytes_{0L}; + uint64_t reservationBytes_{0L}; +}; + } // namespace gluten diff --git a/cpp/core/memory/MemoryAllocator.cc b/cpp/core/memory/MemoryAllocator.cc index 6bcb9926eb45..ac869219d5c1 100644 --- a/cpp/core/memory/MemoryAllocator.cc +++ b/cpp/core/memory/MemoryAllocator.cc @@ -22,54 +22,38 @@ namespace gluten { bool ListenableMemoryAllocator::allocate(int64_t size, void** out) { - listener_->allocationChanged(size); + updateUsage(size); bool succeed = delegated_->allocate(size, out); if (!succeed) { - listener_->allocationChanged(-size); - } - if (succeed) { - bytes_ += size; - peakBytes_ = std::max(peakBytes_, bytes_.load()); + updateUsage(-size); } return succeed; } bool ListenableMemoryAllocator::allocateZeroFilled(int64_t nmemb, int64_t size, void** out) { - listener_->allocationChanged(size * nmemb); + updateUsage(size * nmemb); bool succeed = delegated_->allocateZeroFilled(nmemb, size, out); if (!succeed) { - listener_->allocationChanged(-size * nmemb); - } - if (succeed) { - bytes_ += size * nmemb; - peakBytes_ = std::max(peakBytes_, bytes_.load()); + updateUsage(-size * nmemb); } return succeed; } bool ListenableMemoryAllocator::allocateAligned(uint64_t alignment, int64_t size, void** out) { - listener_->allocationChanged(size); + updateUsage(size); bool succeed = delegated_->allocateAligned(alignment, size, out); if (!succeed) { - listener_->allocationChanged(-size); - } - if (succeed) { - bytes_ += size; - peakBytes_ = std::max(peakBytes_, bytes_.load()); + updateUsage(-size); } return succeed; } bool ListenableMemoryAllocator::reallocate(void* p, int64_t size, int64_t newSize, void** out) { int64_t diff = newSize - size; - listener_->allocationChanged(diff); + updateUsage(diff); bool succeed = delegated_->reallocate(p, size, newSize, out); if (!succeed) { - listener_->allocationChanged(-diff); - } - if (succeed) { - bytes_ += diff; - peakBytes_ = std::max(peakBytes_, bytes_.load()); + updateUsage(-diff); } return succeed; } @@ -81,38 +65,37 @@ bool ListenableMemoryAllocator::reallocateAligned( int64_t newSize, void** out) { int64_t diff = newSize - size; - listener_->allocationChanged(diff); + updateUsage(diff); bool succeed = delegated_->reallocateAligned(p, alignment, size, newSize, out); if (!succeed) { - listener_->allocationChanged(-diff); - } - if (succeed) { - bytes_ += diff; - peakBytes_ = std::max(peakBytes_, bytes_.load()); + updateUsage(-diff); } return succeed; } bool ListenableMemoryAllocator::free(void* p, int64_t size) { - listener_->allocationChanged(-size); + updateUsage(-size); bool succeed = delegated_->free(p, size); if (!succeed) { - listener_->allocationChanged(size); - } - if (succeed) { - bytes_ -= size; + updateUsage(size); } return succeed; } int64_t ListenableMemoryAllocator::getBytes() const { - return bytes_; + return usedBytes_; } int64_t ListenableMemoryAllocator::peakBytes() const { return peakBytes_; } +void ListenableMemoryAllocator::updateUsage(int64_t size) { + listener_->allocationChanged(size); + usedBytes_ += size; + peakBytes_ = std::max(peakBytes_, usedBytes_); +} + bool StdMemoryAllocator::allocate(int64_t size, void** out) { *out = std::malloc(size); bytes_ += size; diff --git a/cpp/core/memory/MemoryAllocator.h b/cpp/core/memory/MemoryAllocator.h index a322c9190f9b..bc8f9de1815e 100644 --- a/cpp/core/memory/MemoryAllocator.h +++ b/cpp/core/memory/MemoryAllocator.h @@ -68,10 +68,11 @@ class ListenableMemoryAllocator final : public MemoryAllocator { int64_t peakBytes() const override; private: + void updateUsage(int64_t size); MemoryAllocator* delegated_; AllocationListener* listener_; - std::atomic_int64_t bytes_{0}; - int64_t peakBytes_{0}; + uint64_t usedBytes_{0L}; + uint64_t peakBytes_{0L}; }; class StdMemoryAllocator final : public MemoryAllocator { diff --git a/cpp/velox/benchmarks/common/BenchmarkUtils.cc b/cpp/velox/benchmarks/common/BenchmarkUtils.cc index ccec6f3c40b1..a9f6f0838cfa 100644 --- a/cpp/velox/benchmarks/common/BenchmarkUtils.cc +++ b/cpp/velox/benchmarks/common/BenchmarkUtils.cc @@ -18,7 +18,7 @@ #include "BenchmarkUtils.h" #include "compute/VeloxBackend.h" #include "compute/VeloxRuntime.h" -#include "config/GlutenConfig.h" +#include "config/VeloxConfig.h" #include "shuffle/Utils.h" #include "utils/StringUtil.h" #include "velox/dwio/common/Options.h" @@ -38,6 +38,7 @@ std::unordered_map bmConfMap = {{gluten::kSparkBatchSi } // namespace void initVeloxBackend(std::unordered_map& conf) { + conf[gluten::kGlogSeverityLevel] = "0"; gluten::VeloxBackend::create(conf); } diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index f49beaccd264..0584780ad5ab 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -20,6 +20,8 @@ #include "velox/common/memory/MemoryPool.h" #include "velox/exec/MemoryReclaimer.h" +#include "compute/VeloxBackend.h" +#include "config/VeloxConfig.h" #include "memory/ArrowMemoryPool.h" #include "utils/exception.h" @@ -103,9 +105,9 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { } } auto reclaimedFreeBytes = pool->shrink(0); - auto neededBytes = bytes - reclaimedFreeBytes; + auto neededBytes = velox::bits::roundUp(bytes - reclaimedFreeBytes, memoryPoolTransferCapacity_); listener_->allocationChanged(neededBytes); - auto ret = pool->grow(bytes, bytes); + auto ret = pool->grow(reclaimedFreeBytes + neededBytes, bytes); VELOX_CHECK( ret, "{} failed to grow {} bytes, current state {}", @@ -156,8 +158,11 @@ VeloxMemoryManager::VeloxMemoryManager( std::shared_ptr allocator, std::unique_ptr listener) : MemoryManager(), name_(name), listener_(std::move(listener)) { - glutenAlloc_ = std::make_unique(allocator.get(), listener_.get()); - arrowPool_ = std::make_unique(glutenAlloc_.get()); + auto reservationBlockSize = VeloxBackend::get()->getBackendConf()->get( + kMemoryReservationBlockSize, kMemoryReservationBlockSizeDefault); + blockListener_ = std::make_unique(listener_.get(), reservationBlockSize); + listenableAlloc_ = std::make_unique(allocator.get(), blockListener_.get()); + arrowPool_ = std::make_unique(listenableAlloc_.get()); ArbitratorFactoryRegister afr(listener_.get()); velox::memory::MemoryManagerOptions mmOptions{ @@ -169,7 +174,7 @@ VeloxMemoryManager::VeloxMemoryManager( .allocatorCapacity = velox::memory::kMaxMemory, .arbitratorKind = afr.getKind(), .memoryPoolInitCapacity = 0, - .memoryPoolTransferCapacity = 32 << 20, + .memoryPoolTransferCapacity = reservationBlockSize, .memoryReclaimWaitMs = 0}; veloxMemoryManager_ = std::make_unique(mmOptions); @@ -222,7 +227,7 @@ const MemoryUsageStats VeloxMemoryManager::collectMemoryUsageStats() const { stats.set_current(listener_->currentBytes()); stats.set_peak(listener_->peakBytes()); stats.mutable_children()->emplace( - "gluten::MemoryAllocator", collectGlutenAllocatorMemoryUsageStats(glutenAlloc_.get())); + "gluten::MemoryAllocator", collectGlutenAllocatorMemoryUsageStats(listenableAlloc_.get())); stats.mutable_children()->emplace( veloxAggregatePool_->name(), collectVeloxMemoryUsageStats(veloxAggregatePool_.get())); return stats; diff --git a/cpp/velox/memory/VeloxMemoryManager.h b/cpp/velox/memory/VeloxMemoryManager.h index 1e8bcd8c8d5e..3ba5bbf7d25f 100644 --- a/cpp/velox/memory/VeloxMemoryManager.h +++ b/cpp/velox/memory/VeloxMemoryManager.h @@ -60,6 +60,11 @@ class VeloxMemoryManager final : public MemoryManager { void hold() override; + /// Test only + MemoryAllocator* allocator() const { + return listenableAlloc_.get(); + } + AllocationListener* getListener() const { return listener_.get(); } @@ -74,8 +79,9 @@ class VeloxMemoryManager final : public MemoryManager { #endif // This is a listenable allocator used for arrow. - std::unique_ptr glutenAlloc_; + std::unique_ptr listenableAlloc_; std::unique_ptr listener_; + std::unique_ptr blockListener_; std::unique_ptr arrowPool_; std::unique_ptr veloxMemoryManager_; diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt index a5bd5b4f7c9d..29beb69da220 100644 --- a/cpp/velox/tests/CMakeLists.txt +++ b/cpp/velox/tests/CMakeLists.txt @@ -30,12 +30,14 @@ function(add_velox_test TEST_EXEC) else() message(FATAL_ERROR "No sources specified for test ${TEST_NAME}") endif() - add_executable(${TEST_EXEC} ${SOURCES}) + add_executable(${TEST_EXEC} ${SOURCES} ${VELOX_TEST_COMMON_SRCS}) target_include_directories(${TEST_EXEC} PRIVATE ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/src ${VELOX_BUILD_PATH}/_deps/duckdb-src/src/include) - target_link_libraries(${TEST_EXEC} velox GTest::gtest GTest::gtest_main google::glog benchmark::benchmark) + target_link_libraries(${TEST_EXEC} velox_benchmark_common GTest::gtest GTest::gtest_main) gtest_discover_tests(${TEST_EXEC} DISCOVERY_MODE PRE_TEST) endfunction() +set(VELOX_TEST_COMMON_SRCS JsonToProtoConverter.cc FilePathGenerator.cc) + add_velox_test(velox_shuffle_writer_test SOURCES VeloxShuffleWriterTest.cc) # TODO: ORC is not well supported. # add_velox_test(orc_test SOURCES OrcTest.cc) @@ -55,10 +57,8 @@ add_velox_test( SubstraitExtensionCollectorTest.cc VeloxSubstraitRoundTripTest.cc VeloxSubstraitSignatureTest.cc - VeloxToSubstraitTypeTest.cc - FunctionTest.cc - JsonToProtoConverter.cc - FilePathGenerator.cc) -add_velox_test(spark_functions_test SOURCES SparkFunctionTest.cc) + VeloxToSubstraitTypeTest.cc) +add_velox_test(spark_functions_test SOURCES SparkFunctionTest.cc FunctionTest.cc) add_velox_test(execution_ctx_test SOURCES RuntimeTest.cc) +add_velox_test(velox_memory_test SOURCES MemoryManagerTest.cc) add_velox_test(buffer_outputstream_test SOURCES BufferOutputStreamTest.cc) diff --git a/cpp/velox/tests/FunctionTest.cc b/cpp/velox/tests/FunctionTest.cc index 01af301768d8..b55b64ba9811 100644 --- a/cpp/velox/tests/FunctionTest.cc +++ b/cpp/velox/tests/FunctionTest.cc @@ -15,34 +15,33 @@ * limitations under the License. */ +#include "FilePathGenerator.h" #include "JsonToProtoConverter.h" -#include "memory/VeloxMemoryManager.h" #include "velox/common/base/Fs.h" #include "velox/common/base/tests/GTestUtils.h" +#include "velox/core/QueryCtx.h" #include "velox/dwio/common/tests/utils/DataFiles.h" +#include "velox/vector/tests/utils/VectorTestBase.h" +#include "substrait/SubstraitParser.h" #include "substrait/SubstraitToVeloxPlan.h" #include "substrait/TypeUtils.h" #include "substrait/VariantToVectorConverter.h" #include "substrait/VeloxToSubstraitType.h" -#include "FilePathGenerator.h" - -#include "velox/core/QueryCtx.h" - -#include "substrait/SubstraitParser.h" - using namespace facebook::velox; using namespace facebook::velox::test; namespace gluten { -class FunctionTest : public ::testing::Test { +class FunctionTest : public ::testing::Test, public test::VectorTestBase { protected: - std::shared_ptr pool_ = gluten::defaultLeafVeloxMemoryPool(); + static void SetUpTestCase() { + memory::MemoryManager::testingSetInstance({}); + } std::shared_ptr planConverter_ = - std::make_shared(pool_.get()); + std::make_shared(pool()); }; TEST_F(FunctionTest, makeNames) { diff --git a/cpp/velox/tests/MemoryManagerTest.cc b/cpp/velox/tests/MemoryManagerTest.cc new file mode 100644 index 000000000000..f256db1b2bb5 --- /dev/null +++ b/cpp/velox/tests/MemoryManagerTest.cc @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmarks/common/BenchmarkUtils.h" +#include "compute/VeloxBackend.h" +#include "config/GlutenConfig.h" +#include "memory/VeloxMemoryManager.h" +#include "velox/common/base/tests/GTestUtils.h" + +namespace gluten { +using namespace facebook::velox; + +class MockAllocationListener : public gluten::AllocationListener { + public: + void allocationChanged(int64_t diff) override { + currentBytes_ += diff; + peakBytes_ = std::max(peakBytes_, currentBytes_); + } + int64_t currentBytes() override { + return currentBytes_; + } + int64_t peakBytes() override { + return peakBytes_; + } + uint64_t currentBytes_{0L}; + uint64_t peakBytes_{0L}; +}; + +namespace { +static const uint64_t kMB = 1 << 20; +} // namespace + +class MemoryManagerTest : public ::testing::Test { + protected: + static void SetUpTestCase() { + std::unordered_map conf = { + {kMemoryReservationBlockSize, std::to_string(kMemoryReservationBlockSizeDefault)}}; + initVeloxBackend(conf); + } + + void SetUp() override { + vmm_ = std::make_unique("test", stdAllocator_, std::make_unique()); + listener_ = vmm_->getListener(); + allocator_ = vmm_->allocator(); + } + + std::unique_ptr vmm_; + AllocationListener* listener_; + MemoryAllocator* allocator_; + + std::shared_ptr stdAllocator_ = std::make_shared(); + + struct Allocation { + void* buffer; + size_t size; + memory::MemoryPool* pool; + }; +}; + +TEST_F(MemoryManagerTest, memoryPoolWithBlockReseravtion) { + auto pool = vmm_->getLeafMemoryPool(); + std::vector allocations; + std::vector sizes{ + kMemoryReservationBlockSizeDefault - 1 * kMB, kMemoryReservationBlockSizeDefault - 2 * kMB}; + for (const auto& size : sizes) { + auto buf = pool->allocate(size); + allocations.push_back({buf, size, pool.get()}); + } + EXPECT_EQ(listener_->currentBytes(), 2 * kMemoryReservationBlockSizeDefault); + EXPECT_EQ(listener_->peakBytes(), listener_->currentBytes()); + + for (auto& allocation : allocations) { + allocation.pool->free(allocation.buffer, allocation.size); + } + + auto currentBytes = listener_->currentBytes(); + ASSERT_EQ(vmm_->shrink(0), currentBytes); + ASSERT_EQ(listener_->currentBytes(), 0); +} + +TEST_F(MemoryManagerTest, memoryAllocatorWithBlockReservation) { + std::vector allocations; + std::vector sizes{ + kMemoryReservationBlockSizeDefault - 1 * kMB, kMemoryReservationBlockSizeDefault - 2 * kMB}; + for (auto i = 0; i < sizes.size(); i++) { + auto size = sizes[i]; + auto currentBytes = allocator_->getBytes(); + Allocation allocation{.size = size}; + allocator_->allocate(size, &allocation.buffer); + allocations.push_back(allocation); + + EXPECT_EQ(allocator_->getBytes(), currentBytes + size); + EXPECT_EQ(allocator_->peakBytes(), allocator_->getBytes()); + EXPECT_EQ(listener_->currentBytes(), (i + 1) * kMemoryReservationBlockSizeDefault); + EXPECT_EQ(listener_->peakBytes(), listener_->currentBytes()); + } + + auto currentBytes = allocator_->getBytes(); + auto allocation = allocations.back(); + allocations.pop_back(); + allocator_->free(allocation.buffer, allocation.size); + EXPECT_EQ(allocator_->getBytes(), currentBytes - allocation.size); + EXPECT_EQ(listener_->currentBytes(), kMemoryReservationBlockSizeDefault); + + currentBytes = allocator_->getBytes(); + allocation = allocations.back(); + allocations.pop_back(); + allocator_->free(allocation.buffer, allocation.size); + EXPECT_EQ(allocator_->getBytes(), currentBytes - allocation.size); + EXPECT_EQ(listener_->currentBytes(), 0); + + ASSERT_EQ(allocator_->getBytes(), 0); +} + +} // namespace gluten diff --git a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc index 841514261859..3926b22c9c21 100644 --- a/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc +++ b/cpp/velox/tests/Substrait2VeloxPlanConversionTest.cc @@ -19,7 +19,6 @@ #include #include "compute/VeloxPlanConverter.h" -#include "memory/VeloxMemoryManager.h" #include "substrait/SubstraitToVeloxPlan.h" #include "velox/common/base/tests/GTestUtils.h" #include "velox/dwio/common/tests/utils/DataFiles.h" @@ -72,7 +71,7 @@ class Substrait2VeloxPlanConversionTest : public exec::test::HiveConnectorTestBa std::shared_ptr tmpDir_{exec::test::TempDirectoryPath::create()}; std::shared_ptr planConverter_ = std::make_shared( std::vector>(), - gluten::defaultLeafVeloxMemoryPool().get(), + pool(), std::unordered_map()); }; diff --git a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc index 8d605e308d1a..d5eafa1e2b2f 100644 --- a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc +++ b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc @@ -54,9 +54,6 @@ class Substrait2VeloxPlanValidatorTest : public exec::test::HiveConnectorTestBas auto planValidator = std::make_shared(pool_.get(), execCtx.get()); return planValidator->validate(plan); } - - private: - std::shared_ptr memoryPool_{gluten::defaultLeafVeloxMemoryPool()}; }; TEST_F(Substrait2VeloxPlanValidatorTest, group) { diff --git a/cpp/velox/tests/VeloxColumnarBatchSerializerTest.cc b/cpp/velox/tests/VeloxColumnarBatchSerializerTest.cc index 4f1745f81660..ffa6f032ac44 100644 --- a/cpp/velox/tests/VeloxColumnarBatchSerializerTest.cc +++ b/cpp/velox/tests/VeloxColumnarBatchSerializerTest.cc @@ -29,14 +29,12 @@ using namespace facebook::velox; namespace gluten { class VeloxColumnarBatchSerializerTest : public ::testing::Test, public test::VectorTestBase { - protected: - std::shared_ptr arrowPool_ = defaultArrowMemoryPool(); - std::shared_ptr veloxPool_ = defaultLeafVeloxMemoryPool(); - // velox requires the mem manager to be instanced protected: static void SetUpTestCase() { memory::MemoryManager::testingSetInstance({}); } + + std::shared_ptr arrowPool_ = defaultArrowMemoryPool(); }; TEST_F(VeloxColumnarBatchSerializerTest, serialize) { @@ -54,12 +52,12 @@ TEST_F(VeloxColumnarBatchSerializerTest, serialize) { }; auto vector = makeRowVector(children); auto batch = std::make_shared(vector); - auto serializer = std::make_shared(arrowPool_.get(), veloxPool_, nullptr); + auto serializer = std::make_shared(arrowPool_.get(), pool_, nullptr); auto buffer = serializer->serializeColumnarBatches({batch}); ArrowSchema cSchema; exportToArrow(vector, cSchema, ArrowUtils::getBridgeOptions()); - auto deserializer = std::make_shared(arrowPool_.get(), veloxPool_, &cSchema); + auto deserializer = std::make_shared(arrowPool_.get(), pool_, &cSchema); auto deserialized = deserializer->deserialize(const_cast(buffer->data()), buffer->size()); auto deserializedVector = std::dynamic_pointer_cast(deserialized)->getRowVector(); test::assertEqualVectors(vector, deserializedVector); diff --git a/cpp/velox/tests/VeloxColumnarBatchTest.cc b/cpp/velox/tests/VeloxColumnarBatchTest.cc index 559f9f047258..ba66afb40fdf 100644 --- a/cpp/velox/tests/VeloxColumnarBatchTest.cc +++ b/cpp/velox/tests/VeloxColumnarBatchTest.cc @@ -24,12 +24,9 @@ using namespace facebook::velox; namespace gluten { class VeloxColumnarBatchTest : public ::testing::Test, public test::VectorTestBase { protected: - // Velox requires the mem manager to be instanced. static void SetUpTestCase() { memory::MemoryManager::testingSetInstance({}); } - - std::shared_ptr veloxPool_ = defaultLeafVeloxMemoryPool(); }; TEST_F(VeloxColumnarBatchTest, flattenTruncatedVector) { @@ -43,7 +40,7 @@ TEST_F(VeloxColumnarBatchTest, flattenTruncatedVector) { // First, make a row vector with the mapKeys and mapValues as children. // Make the row vector size less than the children size. auto input = std::make_shared( - veloxPool_.get(), + pool(), ROW({INTEGER(), BIGINT(), MAP(INTEGER(), BIGINT())}), nullptr, inputSize, @@ -54,7 +51,7 @@ TEST_F(VeloxColumnarBatchTest, flattenTruncatedVector) { // Allocate a dummy indices and wrap the original mapVector with it as a dictionary, to force it get decoded in // flattenVector. - auto indices = allocateIndices(childSize, veloxPool_.get()); + auto indices = allocateIndices(childSize, pool()); auto* rawIndices = indices->asMutable(); for (vector_size_t i = 0; i < childSize; i++) { rawIndices[i] = i; diff --git a/cpp/velox/tests/VeloxColumnarToRowTest.cc b/cpp/velox/tests/VeloxColumnarToRowTest.cc index 1769fa7abfde..2309e6e1cd30 100644 --- a/cpp/velox/tests/VeloxColumnarToRowTest.cc +++ b/cpp/velox/tests/VeloxColumnarToRowTest.cc @@ -15,7 +15,6 @@ * limitations under the License. */ -#include "jni/JniError.h" #include "memory/VeloxColumnarBatch.h" #include "memory/VeloxMemoryManager.h" #include "operators/serializer/VeloxColumnarToRowConverter.h" @@ -35,7 +34,7 @@ class VeloxColumnarToRowTest : public ::testing::Test, public test::VectorTestBa } void testRowBufferAddr(velox::RowVectorPtr vector, uint8_t* expectArr, int32_t expectArrSize) { - auto columnarToRowConverter = std::make_shared(veloxPool_); + auto columnarToRowConverter = std::make_shared(pool_); auto cb = std::make_shared(vector); columnarToRowConverter->convert(cb); @@ -45,9 +44,6 @@ class VeloxColumnarToRowTest : public ::testing::Test, public test::VectorTestBa ASSERT_EQ(*(address + i), *(expectArr + i)); } } - - private: - std::shared_ptr veloxPool_ = defaultLeafVeloxMemoryPool(); }; TEST_F(VeloxColumnarToRowTest, Buffer_int8_int16) { diff --git a/cpp/velox/tests/VeloxRowToColumnarTest.cc b/cpp/velox/tests/VeloxRowToColumnarTest.cc index 62f809ac2c07..93f780ca3a38 100644 --- a/cpp/velox/tests/VeloxRowToColumnarTest.cc +++ b/cpp/velox/tests/VeloxRowToColumnarTest.cc @@ -55,9 +55,6 @@ class VeloxRowToColumnarTest : public ::testing::Test, public test::VectorTestBa auto vp = std::dynamic_pointer_cast(cb)->getRowVector(); velox::test::assertEqualVectors(vector, vp); } - - private: - std::shared_ptr arrowPool_ = defaultArrowMemoryPool(); }; TEST_F(VeloxRowToColumnarTest, allTypes) { diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java index 230a7342e87e..0d1a0c5aec41 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java @@ -16,7 +16,6 @@ */ package org.apache.gluten.memory.nmm; -import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; import org.apache.gluten.memory.alloc.NativeMemoryAllocators; import org.apache.gluten.memory.memtarget.KnownNameAndStats; @@ -46,12 +45,8 @@ private NativeMemoryManager( public static NativeMemoryManager create(String name, ReservationListener listener) { long allocatorId = NativeMemoryAllocators.getDefault().get().getNativeInstanceId(); - long reservationBlockSize = GlutenConfig.getConf().memoryReservationBlockSize(); return new NativeMemoryManager( - name, - create( - BackendsApiManager.getBackendName(), name, allocatorId, reservationBlockSize, listener), - listener); + name, create(BackendsApiManager.getBackendName(), name, allocatorId, listener), listener); } public long getNativeInstanceHandle() { @@ -81,11 +76,7 @@ public void hold() { private static native long shrink(long memoryManagerId, long size); private static native long create( - String backendType, - String name, - long allocatorId, - long reservationBlockSize, - ReservationListener listener); + String backendType, String name, long allocatorId, ReservationListener listener); private static native void release(long memoryManagerId); From 864c6bb666684dac673038beb227579ed7eb0e6a Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Mon, 20 May 2024 19:27:43 +0800 Subject: [PATCH 111/402] [VL] Daily Update Velox Version (2024_05_20) (#5807) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 33a82ca57d8e..c37933d10857 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_17 +VELOX_BRANCH=2024_05_20 VELOX_HOME="" #Set on run gluten on HDFS From 6195e5d67e010d6f7d05921bc77f52a6d1349472 Mon Sep 17 00:00:00 2001 From: zhaokuo Date: Mon, 20 May 2024 20:19:59 +0800 Subject: [PATCH 112/402] [VL][Minor] Fix warnings caused by -Wunused-but-set-variable --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index b50f9bd346be..34ba6057c15f 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -2158,8 +2158,8 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( upperBound = getMax(); } - bool lowerUnbounded = true; - bool upperUnbounded = true; + [[maybe_unused]] bool lowerUnbounded = true; + [[maybe_unused]] bool upperUnbounded = true; bool lowerExclusive = false; bool upperExclusive = false; From d74fc97cf941759c79f440b0df5c5071655b984e Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Mon, 20 May 2024 20:33:06 +0800 Subject: [PATCH 113/402] [VL] Enable rint function (#5791) [VL] Enable rint function. --- .../org/apache/gluten/utils/CHExpressionUtil.scala | 3 ++- .../execution/ScalarFunctionsValidateSuite.scala | 12 ++++++++++++ docs/velox-backend-support-progress.md | 2 +- .../gluten/expression/ExpressionMappings.scala | 1 + .../apache/gluten/expression/ExpressionNames.scala | 1 + 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index f593e5faceef..8454b1469009 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -199,6 +199,7 @@ object CHExpressionUtil { UNIX_MICROS -> DefaultValidator(), TIMESTAMP_MILLIS -> DefaultValidator(), TIMESTAMP_MICROS -> DefaultValidator(), - FLATTEN -> DefaultValidator() + FLATTEN -> DefaultValidator(), + RINT -> DefaultValidator() ) } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index e88e9699a9d0..cba8b6207a7f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -985,4 +985,16 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { checkGlutenOperatorMatch[ProjectExecTransformer] } } + + test("rint") { + withTempPath { + path => + Seq(1.2, 1.5, 1.9).toDF("d").write.parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("double") + runQueryAndCompare("select rint(d) from double") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } } diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 8b640c081da5..b8acce266e9a 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -245,7 +245,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | rand | rand | rand | S | | | | | | | | | | | | | | | | | | | | | rand | rand | rand | | | | | | | | | | | | | | | | | | | | | | random | random | | S | | | | | | | | | | | | | | | | | | | | -| rint | | | | | | | | | | | | | | | | | | | | | | | +| rint | | rint |S | | | | | | | | | | | | | | | | | | | | | round | round | round | S | | | S | S | S | S | S | S | | | | | | | | | | | | | shiftleft | bitwise_left_shift | shiftleft | S | | | S | S | S | S | S | S | | | | | | | | | | | | | shiftright | bitwise_right_shift | shiftright | S | | | S | S | S | S | S | S | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 1592b0b9aa2d..f910c8a98e29 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -148,6 +148,7 @@ object ExpressionMappings { Sig[Remainder](REMAINDER), Sig[Factorial](FACTORIAL), Sig[Rand](RAND), + Sig[Rint](RINT), // PrestoSQL Math functions Sig[Acos](ACOS), Sig[Asin](ASIN), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 735bb7a23f41..1b73b2686952 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -175,6 +175,7 @@ object ExpressionNames { final val REMAINDER = "modulus" final val FACTORIAL = "factorial" final val RAND = "rand" + final val RINT = "rint" // PrestoSQL Math functions final val ACOS = "acos" From e987157f96ca7c4b880d25c2c82d65295a5ea67e Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 21 May 2024 07:51:31 +0800 Subject: [PATCH 114/402] [VL][CI] disable SF30 tpc tests on GHA (#5818) Will move these tests back to Internal CI to reduce the github runner usage Signed-off-by: Yuan Zhou --- .github/workflows/velox_docker.yml | 102 ++++++++++++++--------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index b7ee3be4145e..88c6c2a241b4 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -338,57 +338,57 @@ jobs: --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ --skip-data-gen --random-kill-tasks - run-tpc-test-ubuntu-sf30: - needs: build-native-lib-centos-7 - strategy: - fail-fast: false - matrix: - spark: [ "spark-3.4" ] - shard: [ "1/4", "2/4", "3/4", "4/4" ] - runs-on: ubuntu-20.04 - steps: - - name: Maximize build disk space - shell: bash - run: | - df -h - set -euo pipefail - echo "Removing unwanted software... " - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force > /dev/null - df -h - - uses: actions/checkout@v2 - - name: Download All Artifacts - uses: actions/download-artifact@v2 - with: - name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases - - name: Setup java and maven - run: | - sudo apt-get update - sudo apt-get install -y openjdk-8-jdk maven - - name: Set environment variables - run: | - echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - - name: Build for Spark ${{ matrix.spark }} - run: | - cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests - cd $GITHUB_WORKSPACE/tools/gluten-it - mvn -ntp clean install -P${{ matrix.spark }} - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=h -s=30.0 --threads=12 - GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - - name: TPC-H / TPC-DS SF30.0 Parquet local ${{ matrix.spark }} - run: | - cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ - --skip-data-gen --shard=${{ matrix.shard }} \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ - --skip-data-gen --shard=${{ matrix.shard }} + # run-tpc-test-ubuntu-sf30: + # needs: build-native-lib-centos-7 + # strategy: + # fail-fast: false + # matrix: + # spark: [ "spark-3.4" ] + # shard: [ "1/4", "2/4", "3/4", "4/4" ] + # runs-on: ubuntu-20.04 + # steps: + # - name: Maximize build disk space + # shell: bash + # run: | + # df -h + # set -euo pipefail + # echo "Removing unwanted software... " + # sudo rm -rf /usr/share/dotnet + # sudo rm -rf /usr/local/lib/android + # sudo rm -rf /opt/ghc + # sudo rm -rf /opt/hostedtoolcache/CodeQL + # sudo docker image prune --all --force > /dev/null + # df -h + # - uses: actions/checkout@v2 + # - name: Download All Artifacts + # uses: actions/download-artifact@v2 + # with: + # name: velox-native-lib-centos-7-${{github.sha}} + # path: ./cpp/build/releases + # - name: Setup java and maven + # run: | + # sudo apt-get update + # sudo apt-get install -y openjdk-8-jdk maven + # - name: Set environment variables + # run: | + # echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV + # - name: Build for Spark ${{ matrix.spark }} + # run: | + # cd $GITHUB_WORKSPACE/ + # mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests + # cd $GITHUB_WORKSPACE/tools/gluten-it + # mvn -ntp clean install -P${{ matrix.spark }} + # GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=h -s=30.0 --threads=12 + # GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 + # - name: TPC-H / TPC-DS SF30.0 Parquet local ${{ matrix.spark }} + # run: | + # cd tools/gluten-it \ + # && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=h --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ + # --skip-data-gen --shard=${{ matrix.shard }} \ + # && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ + # --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ + # --skip-data-gen --shard=${{ matrix.shard }} run-tpc-test-centos8-uniffle: needs: build-native-lib-centos-7 From ef26a2e04262eafd0e9ce09d9b01c9a6a1c9fb92 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 21 May 2024 08:07:01 +0800 Subject: [PATCH 115/402] [VL] Ensure get(GetArrayItem) function is offloaded (#5789) --- .../clickhouse/CHSparkPlanExecApi.scala | 21 ++----- .../expression/CHExpressionTransformer.scala | 41 ++++++++++++++ .../velox/VeloxSparkPlanExecApi.scala | 56 +++---------------- .../ScalarFunctionsValidateSuite.scala | 17 ++++++ cpp/velox/substrait/SubstraitParser.cc | 1 + .../SubstraitToVeloxPlanValidator.cc | 3 - .../gluten/backendsapi/SparkPlanExecApi.scala | 9 ++- .../ArrayExpressionTransformer.scala | 42 -------------- .../expression/ExpressionConverter.scala | 3 +- .../apache/spark/sql/GlutenQueryTest.scala | 6 +- 10 files changed, 79 insertions(+), 120 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index cb706d817e71..d6e323679a8d 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -21,7 +21,6 @@ import org.apache.gluten.backendsapi.{BackendsApiManager, SparkPlanExecApi} import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ -import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.extension.{CountDistinctWithoutExpand, FallbackBroadcastHashJoin, FallbackBroadcastHashJoinPrepQueryStage, RewriteToDateExpresstionRule} import org.apache.gluten.extension.columnar.AddTransformHintRule import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPreOverrides @@ -62,7 +61,6 @@ import org.apache.spark.sql.extension.{CommonSubexpressionEliminateRule, Rewrite import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch -import com.google.common.collect.Lists import org.apache.commons.lang3.ClassUtils import java.lang.{Long => JLong} @@ -76,21 +74,12 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { override def batchType: Convention.BatchType = CHBatch /** Transform GetArrayItem to Substrait. */ - override def genGetArrayItemExpressionNode( + override def genGetArrayItemTransformer( substraitExprName: String, - functionMap: JMap[String, JLong], - leftNode: ExpressionNode, - rightNode: ExpressionNode, - original: GetArrayItem): ExpressionNode = { - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.left.dataType, original.right.dataType), - FunctionConfig.OPT) - val exprNodes = Lists.newArrayList(leftNode, rightNode) - ExpressionBuilder.makeScalarFunction( - ExpressionBuilder.newScalarFunction(functionMap, functionName), - exprNodes, - ConverterUtils.getTypeNode(original.dataType, original.nullable)) + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Expression): ExpressionTransformer = { + GetArrayItemTransformer(substraitExprName, left, right, original) } override def genProjectExecTransformer( diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala index 98cc4a930d2f..6403471c7414 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala @@ -222,3 +222,44 @@ case class CHRegExpReplaceTransformer( .doTransform(args) } } + +case class GetArrayItemTransformer( + substraitExprName: String, + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Expression) + extends ExpressionTransformerWithOrigin { + + override def doTransform(args: java.lang.Object): ExpressionNode = { + // Ignore failOnError for clickhouse backend + val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] + val leftNode = left.doTransform(args) + var rightNode = right.doTransform(args) + + val getArrayItem = original.asInstanceOf[GetArrayItem] + + // In Spark, the index of getarrayitem starts from 0 + // But in CH, the index of arrayElement starts from 1, besides index argument must + // So we need to do transform: rightNode = add(rightNode, 1) + val addFunctionName = ConverterUtils.makeFuncName( + ExpressionNames.ADD, + Seq(IntegerType, getArrayItem.right.dataType), + FunctionConfig.OPT) + val addFunctionId = ExpressionBuilder.newScalarFunction(functionMap, addFunctionName) + val literalNode = ExpressionBuilder.makeLiteral(1.toInt, IntegerType, false) + rightNode = ExpressionBuilder.makeScalarFunction( + addFunctionId, + Lists.newArrayList(literalNode, rightNode), + ConverterUtils.getTypeNode(getArrayItem.right.dataType, getArrayItem.right.nullable)) + + val functionName = ConverterUtils.makeFuncName( + substraitExprName, + Seq(getArrayItem.left.dataType, getArrayItem.right.dataType), + FunctionConfig.OPT) + val exprNodes = Lists.newArrayList(leftNode, rightNode) + ExpressionBuilder.makeScalarFunction( + ExpressionBuilder.newScalarFunction(functionMap, functionName), + exprNodes, + ConverterUtils.getTypeNode(getArrayItem.dataType, getArrayItem.nullable)) + } +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 69e56b422561..c30e349529c6 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -22,14 +22,12 @@ import org.apache.gluten.datasource.ArrowConvertorRule import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ -import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} import org.apache.gluten.extension._ import org.apache.gluten.extension.columnar.TransformHints import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.extension.columnar.transition.ConventionFunc.BatchOverride import org.apache.gluten.sql.shims.SparkShimLoader -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, IfThenNode} import org.apache.gluten.vectorized.{ColumnarBatchSerializer, ColumnarBatchSerializeResult} import org.apache.spark.{ShuffleDependency, SparkException} @@ -63,14 +61,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarBatch -import com.google.common.collect.Lists import org.apache.commons.lang3.ClassUtils import javax.ws.rs.core.UriBuilder -import java.lang.{Long => JLong} -import java.util.{Map => JMap} - import scala.collection.mutable.ListBuffer class VeloxSparkPlanExecApi extends SparkPlanExecApi { @@ -91,49 +85,13 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { VeloxBatch } - /** - * Transform GetArrayItem to Substrait. - * - * arrCol[index] => IF(index < 0, null, ElementAt(arrCol, index + 1)) - */ - override def genGetArrayItemExpressionNode( + /** Transform GetArrayItem to Substrait. */ + override def genGetArrayItemTransformer( substraitExprName: String, - functionMap: JMap[String, JLong], - leftNode: ExpressionNode, - rightNode: ExpressionNode, - original: GetArrayItem): ExpressionNode = { - if (original.dataType.isInstanceOf[DecimalType]) { - val decimalType = original.dataType.asInstanceOf[DecimalType] - val precision = decimalType.precision - if (precision > 18) { - throw new GlutenNotSupportException( - "GetArrayItem not support decimal precision more than 18") - } - } - // ignore origin substraitExprName - val functionName = ConverterUtils.makeFuncName( - ExpressionMappings.expressionsMap(classOf[ElementAt]), - Seq(original.dataType), - FunctionConfig.OPT) - val exprNodes = Lists.newArrayList(leftNode, rightNode) - val resultNode = ExpressionBuilder.makeScalarFunction( - ExpressionBuilder.newScalarFunction(functionMap, functionName), - exprNodes, - ConverterUtils.getTypeNode(original.dataType, original.nullable)) - val nullNode = ExpressionBuilder.makeLiteral(null, original.dataType, false) - val lessThanFuncId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - ExpressionNames.LESS_THAN, - Seq(original.right.dataType, IntegerType), - FunctionConfig.OPT)) - // right node already add 1 - val literalNode = ExpressionBuilder.makeLiteral(1.toInt, IntegerType, false) - val lessThanFuncNode = ExpressionBuilder.makeScalarFunction( - lessThanFuncId, - Lists.newArrayList(rightNode, literalNode), - ConverterUtils.getTypeNode(BooleanType, true)) - new IfThenNode(Lists.newArrayList(lessThanFuncNode), Lists.newArrayList(nullNode), resultNode) + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Expression): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(left, right), original) } /** Transform NaNvl to Substrait. */ @@ -521,7 +479,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { resultAttrs: Seq[Attribute], child: SparkPlan, evalType: Int): SparkPlan = { - new ColumnarArrowEvalPythonExec(udfs, resultAttrs, child, evalType) + ColumnarArrowEvalPythonExec(udfs, resultAttrs, child, evalType) } /** diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index cba8b6207a7f..d0df35b64e25 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -978,6 +978,23 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + testWithSpecifiedSparkVersion("get", Some("3.4")) { + withTempPath { + path => + Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null) + .toDF("value") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl") + + runQueryAndCompare( + "select get(value, 0), get(value, 1), get(value, 2), get(value, 3) from array_tbl;") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("length") { runQueryAndCompare( "select length(c_comment), length(cast(c_comment as binary))" + diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 5a08d83337ec..6f221b78e9ac 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -401,6 +401,7 @@ std::unordered_map SubstraitParser::substraitVeloxFunc {"forall", "all_match"}, {"exists", "any_match"}, {"negative", "unaryminus"}, + {"get_array_item", "get"}, {"arrays_zip", "zip"}}; const std::unordered_map SubstraitParser::typeMap_ = { diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index 51f39a3abdbe..0b08ca20517b 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -194,9 +194,6 @@ bool SubstraitToVeloxPlanValidator::validateScalarFunction( } else if (name == "map_from_arrays") { LOG_VALIDATION_MSG("map_from_arrays is not supported."); return false; - } else if (name == "get_array_item") { - LOG_VALIDATION_MSG("get_array_item is not supported."); - return false; } else if (name == "concat") { for (const auto& type : types) { if (type.find("struct") != std::string::npos || type.find("map") != std::string::npos || diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index a6228e6715e8..7e72b1758c9a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -187,12 +187,11 @@ trait SparkPlanExecApi { } /** Transform GetArrayItem to Substrait. */ - def genGetArrayItemExpressionNode( + def genGetArrayItemTransformer( substraitExprName: String, - functionMap: JMap[String, JLong], - leftNode: ExpressionNode, - rightNode: ExpressionNode, - original: GetArrayItem): ExpressionNode + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Expression): ExpressionTransformer /** Transform NaNvl to Substrait. */ def genNaNvlTransformer( diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala index 68a464f13222..38f65c17893b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala @@ -16,15 +16,11 @@ */ package org.apache.gluten.expression -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.types._ - -import com.google.common.collect.Lists import scala.collection.JavaConverters._ @@ -55,41 +51,3 @@ case class CreateArrayTransformer( ExpressionBuilder.makeScalarFunction(functionId, childNodes, typeNode) } } - -case class GetArrayItemTransformer( - substraitExprName: String, - left: ExpressionTransformer, - right: ExpressionTransformer, - failOnError: Boolean, - original: GetArrayItem) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - // Ignore failOnError for clickhouse backend - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val leftNode = left.doTransform(args) - var rightNode = right.doTransform(args) - - // In Spark, the index of getarrayitem starts from 0 - // But in CH and velox, the index of arrayElement starts from 1, besides index argument must - // So we need to do transform: rightNode = add(rightNode, 1) - val addFunctionName = ConverterUtils.makeFuncName( - ExpressionNames.ADD, - Seq(IntegerType, original.right.dataType), - FunctionConfig.OPT) - val addFunctionId = ExpressionBuilder.newScalarFunction(functionMap, addFunctionName) - val literalNode = ExpressionBuilder.makeLiteral(1.toInt, IntegerType, false) - rightNode = ExpressionBuilder.makeScalarFunction( - addFunctionId, - Lists.newArrayList(literalNode, rightNode), - ConverterUtils.getTypeNode(original.right.dataType, original.right.nullable)) - - BackendsApiManager.getSparkPlanExecApiInstance.genGetArrayItemExpressionNode( - substraitExprName, - functionMap, - leftNode, - rightNode, - original - ) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index b64a23e860fa..e22a20e0dc4c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -174,11 +174,10 @@ object ExpressionConverter extends SQLConfHelper with Logging { c.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) CreateArrayTransformer(substraitExprName, children, useStringTypeWhenEmpty = true, c) case g: GetArrayItem => - GetArrayItemTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genGetArrayItemTransformer( substraitExprName, replaceWithExpressionTransformerInternal(g.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(g.right, attributeSeq, expressionsMap), - g.failOnError, g ) case c: CreateMap => diff --git a/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala b/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala index 32266f1a6245..35afc731bc2e 100644 --- a/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala +++ b/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala @@ -60,13 +60,13 @@ abstract class GlutenQueryTest extends PlanTest { minSparkVersion: Option[String] = None, maxSparkVersion: Option[String] = None): Boolean = { var shouldRun = true - if (!minSparkVersion.isEmpty) { + if (minSparkVersion.isDefined) { shouldRun = isSparkVersionGE(minSparkVersion.get) - if (!maxSparkVersion.isEmpty) { + if (maxSparkVersion.isDefined) { shouldRun = shouldRun && isSparkVersionLE(maxSparkVersion.get) } } else { - if (!maxSparkVersion.isEmpty) { + if (maxSparkVersion.isDefined) { shouldRun = isSparkVersionLE(maxSparkVersion.get) } } From acf3e6de24cf7b858d2a3b4f9f7e93824d4f4c86 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 21 May 2024 11:01:36 +0800 Subject: [PATCH 116/402] [VL][DOC] Update udf doc (#5814) Update the compile command for build native udf library. --- docs/developers/VeloxNativeUDF.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/developers/VeloxNativeUDF.md b/docs/developers/VeloxNativeUDF.md index 13e0a97d1c1c..b951905931e5 100644 --- a/docs/developers/VeloxNativeUDF.md +++ b/docs/developers/VeloxNativeUDF.md @@ -137,7 +137,16 @@ You can also specify the local or HDFS URIs to the UDF libraries or archives. Lo ## Try the example We provided Velox UDF examples in file [MyUDF.cc](../../cpp/velox/udf/examples/MyUDF.cc) and UDAF examples in file [MyUDAF.cc](../../cpp/velox/udf/examples/MyUDAF.cc). -After building gluten cpp, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ +You need to build the gluten cpp project with `--build_example=ON` to get the example libraries. + +```shell +## compile Gluten cpp module +cd /path/to/gluten/cpp +## if you use custom velox_home, make sure specified here by --velox_home +./compile.sh --build_velox_backend=ON --build_examples=ON +``` + +Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ Start spark-shell or spark-sql with below configuration @@ -157,16 +166,16 @@ or Run query. The functions `myudf1` and `myudf2` increment the input value by a constant of 5 ``` -select myudf1(1), myudf2(100L) +select myudf1(100L), myudf2(1) ``` The output from spark-shell will be like ``` -+----------------+------------------+ -|udfexpression(1)|udfexpression(100)| -+----------------+------------------+ -| 6| 105| -+----------------+------------------+ ++------------------+----------------+ +|udfexpression(100)|udfexpression(1)| ++------------------+----------------+ +| 105| 6| ++------------------+----------------+ ``` From ebd9f9f96aff26e56d16a76d994e357c1880c6da Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Tue, 21 May 2024 11:59:16 +0800 Subject: [PATCH 117/402] [VL] Daily Update Velox Version (2024_05_21) (#5819) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index c37933d10857..70b3a9b09987 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_20 +VELOX_BRANCH=2024_05_21 VELOX_HOME="" #Set on run gluten on HDFS From ce92b805fe6fc34c463b40dee42f5a7b83525e6c Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Tue, 21 May 2024 12:42:51 +0800 Subject: [PATCH 118/402] [GLUTEN-5773][VL] Update aws-sdk-cpp version to 1.11.285 (from 1.11.169) (#5774) --- dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch | 12 --- .../ports/aws-sdk-cpp/fix_find_curl.patch | 31 ++++++++ dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake | 18 +++-- dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json | 2 +- dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json | 73 +++++++++++++++++-- 5 files changed, 107 insertions(+), 29 deletions(-) delete mode 100644 dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch create mode 100644 dev/vcpkg/ports/aws-sdk-cpp/fix_find_curl.patch diff --git a/dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch b/dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch deleted file mode 100644 index be4511ada4ef..000000000000 --- a/dev/vcpkg/ports/aws-sdk-cpp/fix-header.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/aws-cpp-sdk-core/include/aws/core/Aws.h b/src/aws-cpp-sdk-core/include/aws/core/Aws.h -index 5c27e75a84c..d221af2039b 100644 ---- a/src/aws-cpp-sdk-core/include/aws/core/Aws.h -+++ b/src/aws-cpp-sdk-core/include/aws/core/Aws.h -@@ -12,6 +12,7 @@ - #include - #include - #include -+#include - #include - #include - diff --git a/dev/vcpkg/ports/aws-sdk-cpp/fix_find_curl.patch b/dev/vcpkg/ports/aws-sdk-cpp/fix_find_curl.patch new file mode 100644 index 000000000000..6f32da02597b --- /dev/null +++ b/dev/vcpkg/ports/aws-sdk-cpp/fix_find_curl.patch @@ -0,0 +1,31 @@ +diff --git a/cmake/external_dependencies.cmake b/cmake/external_dependencies.cmake +index acf16c0..3a49fb4 100644 +--- a/cmake/external_dependencies.cmake ++++ b/cmake/external_dependencies.cmake +@@ -80,23 +80,12 @@ if(NOT NO_HTTP_CLIENT AND NOT USE_CRT_HTTP_CLIENT) + set(BUILD_CURL 1) + message(STATUS " Building Curl as part of AWS SDK") + else() +- include(FindCURL) ++ find_package(CURL REQUIRED) + if(NOT CURL_FOUND) + message(FATAL_ERROR "Could not find curl") ++ else() ++ message(STATUS " Curl library: ${CURL_LIBRARIES}") + endif() +- +- # When built from source using cmake, curl does not include +- # CURL_INCLUDE_DIRS or CURL_INCLUDE_DIRS so we need to use +- # find_package to fix it +- if ("${CURL_INCLUDE_DIRS}" STREQUAL "" AND "${CURL_LIBRARIES}" STREQUAL "") +- message(STATUS "Could not find curl include or library path, falling back to find with config.") +- find_package(CURL) +- set(CURL_LIBRARIES CURL::libcurl) +- else () +- message(STATUS " Curl include directory: ${CURL_INCLUDE_DIRS}") +- List(APPEND EXTERNAL_DEPS_INCLUDE_DIRS ${CURL_INCLUDE_DIRS}) +- set(CLIENT_LIBS ${CURL_LIBRARIES}) +- endif () + set(CLIENT_LIBS_ABSTRACT_NAME curl) + message(STATUS " Curl target link: ${CURL_LIBRARIES}") + endif() diff --git a/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake b/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake index ac3f2292b8c4..71e7014868c9 100644 --- a/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake +++ b/dev/vcpkg/ports/aws-sdk-cpp/portfile.cmake @@ -4,18 +4,18 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO aws/aws-sdk-cpp REF "${VERSION}" - SHA512 63de900870e9bec23d42e9458e0e9b1579a9e2dc7b0f404eae1b0dd406898b6d6841c5e2f498710b3828f212705437da3a2fe94813a6c3a842945100a05ae368 + SHA512 826be806ddd87eb452f97df70b19df4194e984775408d8f99246244b6949abcab583e4cbe1ae3bc5d61f3c78267d0e75ea9e69956188ab12e0318344a4314591 PATCHES patch-relocatable-rpath.patch fix-aws-root.patch lock-curl-http-and-tls-settings.patch fix-awsmigrationhub-build.patch - fix-header.patch + fix_find_curl.patch ) string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) -set(EXTRA_ARGS) +set(EXTRA_ARGS "") if(VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) set(rpath "@loader_path") elseif (VCPKG_TARGET_IS_ANDROID) @@ -31,6 +31,7 @@ else() set(rpath "\$ORIGIN") endif() +string(REPLACE "awsmigrationhub" "AWSMigrationHub" targets "${FEATURES}") vcpkg_cmake_configure( SOURCE_PATH "${SOURCE_PATH}" DISABLE_PARALLEL_CONFIGURE @@ -39,7 +40,7 @@ vcpkg_cmake_configure( "-DENABLE_UNITY_BUILD=ON" "-DENABLE_TESTING=OFF" "-DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}" - "-DBUILD_ONLY=${FEATURES}" + "-DBUILD_ONLY=${targets}" "-DBUILD_DEPS=OFF" "-DBUILD_SHARED_LIBS=OFF" "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" @@ -48,10 +49,11 @@ vcpkg_cmake_configure( ) vcpkg_cmake_install() -foreach(TARGET IN LISTS FEATURES) - vcpkg_cmake_config_fixup(PACKAGE_NAME "aws-cpp-sdk-${TARGET}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) +foreach(TARGET IN LISTS targets) + string(TOLOWER "aws-cpp-sdk-${TARGET}" package) + vcpkg_cmake_config_fixup(PACKAGE_NAME "${package}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) endforeach() -vcpkg_cmake_config_fixup(PACKAGE_NAME "AWSSDK" CONFIG_PATH "lib/cmake/AWSSDK") +vcpkg_cmake_config_fixup(PACKAGE_NAME "awssdk" CONFIG_PATH "lib/cmake/AWSSDK") vcpkg_copy_pdbs() @@ -81,7 +83,7 @@ file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/nuget" ) -if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) +if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") file(GLOB LIB_FILES ${CURRENT_PACKAGES_DIR}/bin/*.lib) if(LIB_FILES) file(COPY ${LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/lib) diff --git a/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json index a618a77d864b..3eb486f732fe 100644 --- a/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json +++ b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.in.json @@ -1,6 +1,6 @@ { "name": "aws-sdk-cpp", - "version": "1.11.160", + "version": "1.11.215", "port-version": 1, "description": "AWS SDK for C++", "homepage": "https://github.com/aws/aws-sdk-cpp", diff --git a/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json index 138894a9eec9..93df56095614 100644 --- a/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json +++ b/dev/vcpkg/ports/aws-sdk-cpp/vcpkg.json @@ -1,8 +1,8 @@ { "$note": "Automatically generated by generateFeatures.ps1", "name": "aws-sdk-cpp", - "version": "1.11.169", - "port-version": 2, + "version": "1.11.285", + "port-version": 1, "description": "AWS SDK for C++", "homepage": "https://github.com/aws/aws-sdk-cpp", "license": "Apache-2.0", @@ -143,6 +143,9 @@ "awstransfer": { "description": "C++ SDK for the AWS awstransfer service" }, + "b2bi": { + "description": "C++ SDK for the AWS b2bi service" + }, "backup": { "description": "C++ SDK for the AWS backup service" }, @@ -155,6 +158,21 @@ "batch": { "description": "C++ SDK for the AWS batch service" }, + "bcm-data-exports": { + "description": "C++ SDK for the AWS bcm-data-exports service" + }, + "bedrock": { + "description": "C++ SDK for the AWS bedrock service" + }, + "bedrock-agent": { + "description": "C++ SDK for the AWS bedrock-agent service" + }, + "bedrock-agent-runtime": { + "description": "C++ SDK for the AWS bedrock-agent-runtime service" + }, + "bedrock-runtime": { + "description": "C++ SDK for the AWS bedrock-runtime service" + }, "billingconductor": { "description": "C++ SDK for the AWS billingconductor service" }, @@ -188,6 +206,9 @@ "cleanrooms": { "description": "C++ SDK for the AWS cleanrooms service" }, + "cleanroomsml": { + "description": "C++ SDK for the AWS cleanroomsml service" + }, "cloud9": { "description": "C++ SDK for the AWS cloud9 service" }, @@ -203,6 +224,9 @@ "cloudfront": { "description": "C++ SDK for the AWS cloudfront service" }, + "cloudfront-keyvaluestore": { + "description": "C++ SDK for the AWS cloudfront-keyvaluestore service" + }, "cloudhsm": { "description": "C++ SDK for the AWS cloudhsm service" }, @@ -296,6 +320,9 @@ "controltower": { "description": "C++ SDK for the AWS controltower service" }, + "cost-optimization-hub": { + "description": "C++ SDK for the AWS cost-optimization-hub service" + }, "cur": { "description": "C++ SDK for the AWS cur service" }, @@ -314,6 +341,9 @@ "datasync": { "description": "C++ SDK for the AWS datasync service" }, + "datazone": { + "description": "C++ SDK for the AWS datazone service" + }, "dax": { "description": "C++ SDK for the AWS dax service" }, @@ -377,6 +407,9 @@ "eks": { "description": "C++ SDK for the AWS eks service" }, + "eks-auth": { + "description": "C++ SDK for the AWS eks-auth service" + }, "elastic-inference": { "description": "C++ SDK for the AWS elastic-inference service" }, @@ -449,15 +482,15 @@ "frauddetector": { "description": "C++ SDK for the AWS frauddetector service" }, + "freetier": { + "description": "C++ SDK for the AWS freetier service" + }, "fsx": { "description": "C++ SDK for the AWS fsx service" }, "gamelift": { "description": "C++ SDK for the AWS gamelift service" }, - "gamesparks": { - "description": "C++ SDK for the AWS gamesparks service" - }, "glacier": { "description": "C++ SDK for the AWS glacier service" }, @@ -519,6 +552,9 @@ "inspector": { "description": "C++ SDK for the AWS inspector service" }, + "inspector-scan": { + "description": "C++ SDK for the AWS inspector-scan service" + }, "inspector2": { "description": "C++ SDK for the AWS inspector2 service" }, @@ -633,6 +669,9 @@ "lambda": { "description": "C++ SDK for the AWS lambda service" }, + "launch-wizard": { + "description": "C++ SDK for the AWS launch-wizard service" + }, "lex": { "description": "C++ SDK for the AWS lex service" }, @@ -678,9 +717,6 @@ "machinelearning": { "description": "C++ SDK for the AWS machinelearning service" }, - "macie": { - "description": "C++ SDK for the AWS macie service" - }, "macie2": { "description": "C++ SDK for the AWS macie2 service" }, @@ -690,9 +726,15 @@ "managedblockchain-query": { "description": "C++ SDK for the AWS managedblockchain-query service" }, + "marketplace-agreement": { + "description": "C++ SDK for the AWS marketplace-agreement service" + }, "marketplace-catalog": { "description": "C++ SDK for the AWS marketplace-catalog service" }, + "marketplace-deployment": { + "description": "C++ SDK for the AWS marketplace-deployment service" + }, "marketplace-entitlement": { "description": "C++ SDK for the AWS marketplace-entitlement service" }, @@ -855,6 +897,12 @@ "proton": { "description": "C++ SDK for the AWS proton service" }, + "qbusiness": { + "description": "C++ SDK for the AWS qbusiness service" + }, + "qconnect": { + "description": "C++ SDK for the AWS qconnect service" + }, "qldb": { "description": "C++ SDK for the AWS qldb service" }, @@ -900,6 +948,9 @@ "rekognition": { "description": "C++ SDK for the AWS rekognition service" }, + "repostspace": { + "description": "C++ SDK for the AWS repostspace service" + }, "resiliencehub": { "description": "C++ SDK for the AWS resiliencehub service" }, @@ -1138,6 +1189,9 @@ "translate": { "description": "C++ SDK for the AWS translate service" }, + "trustedadvisor": { + "description": "C++ SDK for the AWS trustedadvisor service" + }, "verifiedpermissions": { "description": "C++ SDK for the AWS verifiedpermissions service" }, @@ -1177,6 +1231,9 @@ "workspaces": { "description": "C++ SDK for the AWS workspaces service" }, + "workspaces-thin-client": { + "description": "C++ SDK for the AWS workspaces-thin-client service" + }, "workspaces-web": { "description": "C++ SDK for the AWS workspaces-web service" }, From 683232b746263ac6d720a4bf3c61474fe3d39e1a Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Tue, 21 May 2024 15:18:08 +0800 Subject: [PATCH 119/402] [CORE] Refactor ExpressionTransformer (#5796) --- .../clickhouse/CHSparkPlanExecApi.scala | 20 +++ .../clickhouse/CHTransformerApi.scala | 12 -- .../expression/CHExpressionTransformer.scala | 44 ++--- .../velox/VeloxSparkPlanExecApi.scala | 26 ++- .../velox/VeloxTransformerApi.scala | 13 -- .../expression/ExpressionTransformer.scala | 48 ++--- .../spark/sql/expression/UDFResolver.scala | 25 +-- .../gluten/backendsapi/SparkPlanExecApi.scala | 79 ++------- .../gluten/backendsapi/TransformerApi.scala | 7 - .../ArrayExpressionTransformer.scala | 21 +-- .../BoundReferenceTransformer.scala | 29 --- .../expression/ConditionalTransformer.scala | 13 +- .../DateTimeExpressionsTransformer.scala | 164 +++-------------- .../expression/DecimalRoundTransformer.scala | 27 +-- .../expression/ExpressionConverter.scala | 147 ++++++---------- .../expression/ExpressionTransformer.scala | 71 +++++++- .../GenericExpressionTransformer.scala | 46 ----- .../HashExpressionTransformer.scala | 44 ----- .../JsonTupleExpressionTransformer.scala | 2 +- .../LambdaFunctionTransformer.scala | 22 +-- .../expression/LiteralTransformer.scala | 28 --- .../expression/MapExpressionTransformer.scala | 45 +---- .../NamedExpressionsTransformer.scala | 58 ++---- .../PredicateExpressionTransformer.scala | 82 ++------- .../ScalarSubqueryTransformer.scala | 6 +- .../StringExpressionTransformer.scala | 49 ------ .../StructExpressionTransformer.scala | 54 ------ .../expression/TimestampAddTransformer.scala | 53 ------ .../UnaryExpressionTransformer.scala | 166 +++--------------- .../CustomerExpressionTransformer.scala | 26 +-- 30 files changed, 331 insertions(+), 1096 deletions(-) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/BoundReferenceTransformer.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/expression/TimestampAddTransformer.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index d6e323679a8d..45f90719fa41 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -627,6 +627,15 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { CHSizeExpressionTransformer(substraitExprName, child, original) } + override def genLikeTransformer( + substraitExprName: String, + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Like): ExpressionTransformer = { + // CH backend does not support escapeChar, so skip it here. + GenericExpressionTransformer(substraitExprName, Seq(left, right), original) + } + /** Generate an ExpressionTransformer to transform TruncTimestamp expression for CH. */ override def genTruncTimestampTransformer( substraitExprName: String, @@ -637,6 +646,17 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { CHTruncTimestampTransformer(substraitExprName, format, timestamp, timeZoneId, original) } + override def genDateDiffTransformer( + substraitExprName: String, + endDate: ExpressionTransformer, + startDate: ExpressionTransformer, + original: DateDiff): ExpressionTransformer = { + GenericExpressionTransformer( + substraitExprName, + Seq(LiteralTransformer("day"), startDate, endDate), + original) + } + override def genPosExplodeTransformer( substraitExprName: String, child: ExpressionTransformer, diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index ee46d685c6c1..c75cf4788ba9 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -203,18 +203,6 @@ class CHTransformerApi extends TransformerApi with Logging { } - override def createDateDiffParamList( - start: ExpressionNode, - end: ExpressionNode): Iterable[ExpressionNode] = { - List(ExpressionBuilder.makeStringLiteral("day"), start, end) - } - - override def createLikeParamList( - left: ExpressionNode, - right: ExpressionNode, - escapeChar: ExpressionNode): Iterable[ExpressionNode] = - List(left, right) - override def createCheckOverflowExprNode( args: java.lang.Object, substraitExprName: String, diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala index 6403471c7414..5ca4e02339d0 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala @@ -32,17 +32,12 @@ import java.util.Locale case class CHSizeExpressionTransformer( substraitExprName: String, - child: ExpressionTransformer, + expr: ExpressionTransformer, original: Size) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - // Pass legacyLiteral as second argument in substrait function - val legacyLiteral = new Literal(original.legacySizeOfNull, BooleanType) - val legacyTransformer = new LiteralTransformer(legacyLiteral) - GenericExpressionTransformer(substraitExprName, Seq(child, legacyTransformer), original) - .doTransform(args) - } + extends BinaryExpressionTransformer { + override def left: ExpressionTransformer = expr + // Pass legacyLiteral as second argument in substrait function + override def right: ExpressionTransformer = LiteralTransformer(original.legacySizeOfNull) } case class CHTruncTimestampTransformer( @@ -51,7 +46,8 @@ case class CHTruncTimestampTransformer( timestamp: ExpressionTransformer, timeZoneId: Option[String] = None, original: TruncTimestamp) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = format :: timestamp :: Nil override def doTransform(args: java.lang.Object): ExpressionNode = { // The format must be constant string in the function date_trunc of ch. @@ -126,7 +122,8 @@ case class CHStringTranslateTransformer( matchingExpr: ExpressionTransformer, replaceExpr: ExpressionTransformer, original: StringTranslate) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = srcExpr :: matchingExpr :: replaceExpr :: Nil override def doTransform(args: java.lang.Object): ExpressionNode = { // In CH, translateUTF8 requires matchingExpr and replaceExpr argument have the same length @@ -145,11 +142,7 @@ case class CHStringTranslateTransformer( throw new GlutenNotSupportException(s"$original not supported yet.") } - GenericExpressionTransformer( - substraitExprName, - Seq(srcExpr, matchingExpr, replaceExpr), - original) - .doTransform(args) + super.doTransform(args) } } @@ -158,7 +151,7 @@ case class CHPosExplodeTransformer( child: ExpressionTransformer, original: PosExplode, attributeSeq: Seq[Attribute]) - extends ExpressionTransformerWithOrigin { + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode: ExpressionNode = child.doTransform(args) @@ -200,14 +193,15 @@ case class CHPosExplodeTransformer( case class CHRegExpReplaceTransformer( substraitExprName: String, - children: Seq[ExpressionTransformer], + childrenWithPos: Seq[ExpressionTransformer], original: RegExpReplace) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = childrenWithPos.dropRight(1) override def doTransform(args: java.lang.Object): ExpressionNode = { // In CH: replaceRegexpAll(subject, regexp, rep), which is equivalent // In Spark: regexp_replace(subject, regexp, rep, pos=1) - val posNode = children(3).doTransform(args) + val posNode = childrenWithPos(3).doTransform(args) if ( !posNode.isInstanceOf[IntLiteralNode] || posNode.asInstanceOf[IntLiteralNode].getValue != 1 @@ -215,11 +209,7 @@ case class CHRegExpReplaceTransformer( throw new UnsupportedOperationException(s"$original not supported yet.") } - GenericExpressionTransformer( - substraitExprName, - Seq(children(0), children(1), children(2)), - original) - .doTransform(args) + super.doTransform(args) } } @@ -228,7 +218,7 @@ case class GetArrayItemTransformer( left: ExpressionTransformer, right: ExpressionTransformer, original: Expression) - extends ExpressionTransformerWithOrigin { + extends BinaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { // Ignore failOnError for clickhouse backend diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index c30e349529c6..2d37b118592d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -104,6 +104,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { val condFuncName = ExpressionMappings.expressionsMap(classOf[IsNaN]) val newExpr = If(condExpr, original.right, original.left) IfTransformer( + substraitExprName, GenericExpressionTransformer(condFuncName, Seq(left), condExpr), right, left, @@ -117,7 +118,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { original: Uuid): ExpressionTransformer = { GenericExpressionTransformer( substraitExprName, - Seq(LiteralTransformer(Literal(original.randomSeed.get))), + Seq(LiteralTransformer(original.randomSeed.get)), original) } @@ -243,6 +244,17 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(child), expr) } + override def genLikeTransformer( + substraitExprName: String, + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Like): ExpressionTransformer = { + GenericExpressionTransformer( + substraitExprName, + Seq(left, right, LiteralTransformer(original.escapeChar)), + original) + } + /** Transform make_timestamp to Substrait. */ override def genMakeTimestampTransformer( substraitExprName: String, @@ -251,6 +263,14 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, children, expr) } + override def genDateDiffTransformer( + substraitExprName: String, + endDate: ExpressionTransformer, + startDate: ExpressionTransformer, + original: DateDiff): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(endDate, startDate), original) + } + /** * Generate FilterExecTransformer. * @@ -419,7 +439,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { override def genHashExpressionTransformer( substraitExprName: String, exprs: Seq[ExpressionTransformer], - original: Expression): ExpressionTransformer = { + original: HashExpression[_]): ExpressionTransformer = { VeloxHashExpressionTransformer(substraitExprName, exprs, original) } @@ -612,7 +632,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { childTransformer: ExpressionTransformer, ordinal: Int, original: GetStructField): ExpressionTransformer = { - VeloxGetStructFieldTransformer(substraitExprName, childTransformer, ordinal, original) + VeloxGetStructFieldTransformer(substraitExprName, childTransformer, original) } /** diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index 33f612440883..aadfcd9b7d1e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -63,19 +63,6 @@ class VeloxTransformerApi extends TransformerApi with Logging { // TODO: IMPLEMENT SPECIAL PROCESS FOR VELOX BACKEND } - override def createDateDiffParamList( - start: ExpressionNode, - end: ExpressionNode): Iterable[ExpressionNode] = { - List(end, start) - } - - override def createLikeParamList( - left: ExpressionNode, - right: ExpressionNode, - escapeChar: ExpressionNode): Iterable[ExpressionNode] = { - List(left, right, escapeChar) - } - override def createCheckOverflowExprNode( args: java.lang.Object, substraitExprName: String, diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala index da8433fa2e48..0f0eb2969f7e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala @@ -24,8 +24,6 @@ import org.apache.gluten.substrait.expression._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types.{IntegerType, LongType} -import com.google.common.collect.Lists - import java.lang.{Integer => JInteger, Long => JLong} import java.util.{ArrayList => JArrayList, HashMap => JHashMap} @@ -35,7 +33,7 @@ case class VeloxAliasTransformer( substraitExprName: String, child: ExpressionTransformer, original: Expression) - extends ExpressionTransformerWithOrigin { + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { child.doTransform(args) @@ -46,36 +44,25 @@ case class VeloxNamedStructTransformer( substraitExprName: String, original: CreateNamedStruct, attributeSeq: Seq[Attribute]) - extends ExpressionTransformerWithOrigin { - override def doTransform(args: Object): ExpressionNode = { - val expressionNodes = Lists.newArrayList[ExpressionNode]() - original.valExprs.foreach( - child => - expressionNodes.add( - replaceWithExpressionTransformer(child, attributeSeq).doTransform(args))) - val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - val functionName = ConverterUtils - .makeFuncName(substraitExprName, Seq(original.dataType), FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = { + original.valExprs.map(replaceWithExpressionTransformer(_, attributeSeq)) } } case class VeloxGetStructFieldTransformer( substraitExprName: String, - childTransformer: ExpressionTransformer, - ordinal: Int, + child: ExpressionTransformer, original: GetStructField) - extends ExpressionTransformerWithOrigin { + extends UnaryExpressionTransformer { override def doTransform(args: Object): ExpressionNode = { - val childNode = childTransformer.doTransform(args) + val childNode = child.doTransform(args) childNode match { case node: StructLiteralNode => - node.getFieldLiteral(ordinal) + node.getFieldLiteral(original.ordinal) case node: SelectionNode => // Append the nested index to selection node. - node.addNestedChildIdx(JInteger.valueOf(ordinal)) + node.addNestedChildIdx(JInteger.valueOf(original.ordinal)) case other => throw new GlutenNotSupportException(s"$other is not supported.") } @@ -84,9 +71,10 @@ case class VeloxGetStructFieldTransformer( case class VeloxHashExpressionTransformer( substraitExprName: String, - exps: Seq[ExpressionTransformer], - original: Expression) - extends ExpressionTransformerWithOrigin { + children: Seq[ExpressionTransformer], + original: HashExpression[_]) + extends ExpressionTransformer { + override def doTransform(args: java.lang.Object): ExpressionNode = { // As of Spark 3.3, there are 3 kinds of HashExpression. // HiveHash is not supported in native backend and will fail native validation. @@ -101,7 +89,7 @@ case class VeloxHashExpressionTransformer( val nodes = new JArrayList[ExpressionNode]() // Seed as the first argument nodes.add(seedNode) - exps.foreach( + children.foreach( expression => { nodes.add(expression.doTransform(args)) }) @@ -121,7 +109,9 @@ case class VeloxStringSplitTransformer( regexExpr: ExpressionTransformer, limitExpr: ExpressionTransformer, original: StringSplit) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { + // TODO: split function support limit arg + override def children: Seq[ExpressionTransformer] = srcExpr :: regexExpr :: Nil override def doTransform(args: java.lang.Object): ExpressionNode = { if ( @@ -139,8 +129,6 @@ case class VeloxStringSplitTransformer( s"$original supported single-length regex and negative limit, but given $limit and $regex") } - // TODO: split function support limit arg - GenericExpressionTransformer(substraitExprName, Seq(srcExpr, regexExpr), original) - .doTransform(args) + super.doTransform(args) } } diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index c34c1ae7f121..ec98e98f1c6e 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -18,9 +18,7 @@ package org.apache.spark.sql.expression import org.apache.gluten.backendsapi.velox.VeloxBackendSettings import org.apache.gluten.exception.GlutenException -import org.apache.gluten.expression.{ConverterUtils, ExpressionTransformer, ExpressionType, Transformable} -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} +import org.apache.gluten.expression.{ConverterUtils, ExpressionTransformer, ExpressionType, GenericExpressionTransformer, Transformable} import org.apache.gluten.udf.UdfJniWrapper import org.apache.gluten.vectorized.JniWorkspace @@ -37,8 +35,6 @@ import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.types.{DataType, StructField, StructType} import org.apache.spark.util.Utils -import com.google.common.collect.Lists - import java.io.File import java.net.URI import java.nio.file.{Files, FileVisitOption, Paths} @@ -112,24 +108,7 @@ case class UDFExpression( ": getTransformer called before children transformer initialized.") } - val localDataType = dataType - new ExpressionTransformer { - override def doTransform(args: Object): ExpressionNode = { - val transformers = childrenTransformers.map(_.doTransform(args)) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(name, children.map(_.dataType), FunctionConfig.REQ)) - - val typeNode = ConverterUtils.getTypeNode(dataType, nullable) - ExpressionBuilder.makeScalarFunction( - functionId, - Lists.newArrayList(transformers: _*), - typeNode) - } - - override def dataType: DataType = localDataType - } + GenericExpressionTransformer(name, childrenTransformers, this) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 7e72b1758c9a..69777f77a561 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -214,7 +214,7 @@ trait SparkPlanExecApi { throw new GlutenNotSupportException("try_add is not supported") } - def genTryAddTransformer( + def genTryEvalTransformer( substraitExprName: String, child: ExpressionTransformer, original: TryEval): ExpressionTransformer = { @@ -286,9 +286,7 @@ trait SparkPlanExecApi { substraitExprName: String, child: ExpressionTransformer, original: PosExplode, - attributeSeq: Seq[Attribute]): ExpressionTransformer = { - PosExplodeTransformer(substraitExprName, child, original, attributeSeq) - } + attributeSeq: Seq[Attribute]): ExpressionTransformer /** Transform make_timestamp to Substrait. */ def genMakeTimestampTransformer( @@ -427,7 +425,7 @@ trait SparkPlanExecApi { childTransformer: ExpressionTransformer, ordinal: Int, original: GetStructField): ExpressionTransformer = { - GetStructFieldTransformer(substraitExprName, childTransformer, ordinal, original) + GetStructFieldTransformer(substraitExprName, childTransformer, original) } def genNamedStructTransformer( @@ -438,13 +436,6 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, children, original) } - def genMd5Transformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Md5): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(child), original) - } - def genStringTranslateTransformer( substraitExprName: String, srcExpr: ExpressionTransformer, @@ -457,38 +448,6 @@ trait SparkPlanExecApi { original) } - def genStringLocateTransformer( - substraitExprName: String, - first: ExpressionTransformer, - second: ExpressionTransformer, - third: ExpressionTransformer, - original: StringLocate): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(first, second, third), original) - } - - /** - * Generate an ExpressionTransformer to transform Sha2 expression. Sha2Transformer is the default - * implementation. - */ - def genSha2Transformer( - substraitExprName: String, - left: ExpressionTransformer, - right: ExpressionTransformer, - original: Sha2): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(left, right), original) - } - - /** - * Generate an ExpressionTransformer to transform Sha1 expression. Sha1Transformer is the default - * implementation. - */ - def genSha1Transformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Sha1): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(child), original) - } - def genSizeExpressionTransformer( substraitExprName: String, child: ExpressionTransformer, @@ -496,6 +455,12 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(child), original) } + def genLikeTransformer( + substraitExprName: String, + left: ExpressionTransformer, + right: ExpressionTransformer, + original: Like): ExpressionTransformer + /** * Generate an ExpressionTransformer to transform TruncTimestamp expression. * TruncTimestampTransformer is the default implementation. @@ -506,30 +471,22 @@ trait SparkPlanExecApi { timestamp: ExpressionTransformer, timeZoneId: Option[String] = None, original: TruncTimestamp): ExpressionTransformer = { - TruncTimestampTransformer(substraitExprName, format, timestamp, timeZoneId, original) + TruncTimestampTransformer(substraitExprName, format, timestamp, original) } + def genDateDiffTransformer( + substraitExprName: String, + endDate: ExpressionTransformer, + startDate: ExpressionTransformer, + original: DateDiff): ExpressionTransformer + def genCastWithNewChild(c: Cast): Cast = c def genHashExpressionTransformer( substraitExprName: String, exprs: Seq[ExpressionTransformer], - original: Expression): ExpressionTransformer = { - HashExpressionTransformer(substraitExprName, exprs, original) - } - - def genUnixTimestampTransformer( - substraitExprName: String, - timeExp: ExpressionTransformer, - format: ExpressionTransformer, - original: ToUnixTimestamp): ExpressionTransformer = { - ToUnixTimestampTransformer( - substraitExprName, - timeExp, - format, - original.timeZoneId, - original.failOnError, - original) + original: HashExpression[_]): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, exprs, original) } /** Define backend specfic expression mappings. */ diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala index 7a10dc68c8aa..e41df0f2f240 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala @@ -58,13 +58,6 @@ trait TransformerApi { plan.output } - def createDateDiffParamList(start: ExpressionNode, end: ExpressionNode): Iterable[ExpressionNode] - - def createLikeParamList( - left: ExpressionNode, - right: ExpressionNode, - escapeChar: ExpressionNode): Iterable[ExpressionNode] - def createCheckOverflowExprNode( args: java.lang.Object, substraitExprName: String, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala index 38f65c17893b..2a09e039e52c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ArrayExpressionTransformer.scala @@ -17,37 +17,24 @@ package org.apache.gluten.expression import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} +import org.apache.gluten.substrait.expression.ExpressionNode import org.apache.spark.sql.catalyst.expressions._ -import scala.collection.JavaConverters._ - case class CreateArrayTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], - useStringTypeWhenEmpty: Boolean, original: CreateArray) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { // If children is empty, // transformation is only supported when useStringTypeWhenEmpty is false // because ClickHouse and Velox currently doesn't support this config. - if (useStringTypeWhenEmpty && children.isEmpty) { + if (original.useStringTypeWhenEmpty && children.isEmpty) { throw new GlutenNotSupportException(s"$original not supported yet.") } - val childNodes = children.map(_.doTransform(args)).asJava - - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, childNodes, typeNode) + super.doTransform(args) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/BoundReferenceTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/BoundReferenceTransformer.scala deleted file mode 100644 index 2cfced13b7b0..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/BoundReferenceTransformer.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} - -import org.apache.spark.sql.types._ - -case class BoundReferenceTransformer(ordinal: Int, dataType: DataType, nullable: Boolean) - extends ExpressionTransformer { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - ExpressionBuilder.makeSelection(ordinal.asInstanceOf[java.lang.Integer]) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala index 0fdd68511eec..1dffd390639e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ConditionalTransformer.scala @@ -24,10 +24,13 @@ import java.util.{ArrayList => JArrayList} /** A version of substring that supports columnar processing for utf8. */ case class CaseWhenTransformer( + substraitExprName: String, branches: Seq[(ExpressionTransformer, ExpressionTransformer)], elseValue: Option[ExpressionTransformer], - original: Expression) - extends ExpressionTransformerWithOrigin { + original: CaseWhen) + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = + branches.flatMap(b => b._1 :: b._2 :: Nil) ++ elseValue override def doTransform(args: java.lang.Object): ExpressionNode = { // generate branches nodes @@ -48,11 +51,13 @@ case class CaseWhenTransformer( } case class IfTransformer( + substraitExprName: String, predicate: ExpressionTransformer, trueValue: ExpressionTransformer, falseValue: ExpressionTransformer, - original: Expression) - extends ExpressionTransformerWithOrigin { + original: If) + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = predicate :: trueValue :: falseValue :: Nil override def doTransform(args: java.lang.Object): ExpressionNode = { val ifNodes = new JArrayList[ExpressionNode] diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala index 66004291ac4e..505ca33ea74f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/DateTimeExpressionsTransformer.scala @@ -16,140 +16,36 @@ */ package org.apache.gluten.expression -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.types._ - -import com.google.common.collect.Lists - -import java.lang.{Long => JLong} -import java.util.{ArrayList => JArrayList, HashMap => JHashMap} - -import scala.collection.JavaConverters._ /** The extract trait for 'GetDateField' from Date */ case class ExtractDateTransformer( substraitExprName: String, child: ExpressionTransformer, original: Expression) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val childNode = child.doTransform(args) - - val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) + extends BinaryExpressionTransformer { + override def left: ExpressionTransformer = { val dateFieldName = DateTimeExpressionsTransformer.EXTRACT_DATE_FIELD_MAPPING.get(original.getClass) if (dateFieldName.isEmpty) { throw new GlutenNotSupportException(s"$original not supported yet.") } - val fieldNode = ExpressionBuilder.makeStringLiteral(dateFieldName.get) - val expressNodes = Lists.newArrayList(fieldNode, childNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - - ExpressionBuilder.makeScalarFunction(functionId, expressNodes, typeNode) - } -} - -case class DateDiffTransformer( - substraitExprName: String, - endDate: ExpressionTransformer, - startDate: ExpressionTransformer, - original: DateDiff) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val endDateNode = endDate.doTransform(args) - val startDateNode = startDate.doTransform(args) - - val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - Seq(StringType, original.startDate.dataType, original.endDate.dataType), - FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - - val expressionNodes = BackendsApiManager.getTransformerApiInstance.createDateDiffParamList( - startDateNode, - endDateNode) - ExpressionBuilder.makeScalarFunction( - functionId, - expressionNodes.toList.asJava, - ConverterUtils.getTypeNode(original.dataType, original.nullable)) - } -} - -/** - * The failOnError depends on the config for ANSI. ANSI is not supported currently. And timeZoneId - * is passed to backend config. - */ -case class ToUnixTimestampTransformer( - substraitExprName: String, - timeExp: ExpressionTransformer, - format: ExpressionTransformer, - timeZoneId: Option[String], - failOnError: Boolean, - original: ToUnixTimestamp) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val dataTypes = Seq(original.timeExp.dataType, StringType) - val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(substraitExprName, dataTypes)) - - val expressionNodes = new JArrayList[ExpressionNode]() - val timeExpNode = timeExp.doTransform(args) - expressionNodes.add(timeExpNode) - val formatNode = format.doTransform(args) - expressionNodes.add(formatNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) + LiteralTransformer(dateFieldName.get) } + override def right: ExpressionTransformer = child } case class TruncTimestampTransformer( substraitExprName: String, format: ExpressionTransformer, timestamp: ExpressionTransformer, - timeZoneId: Option[String] = None, original: TruncTimestamp) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val timestampNode = timestamp.doTransform(args) - val formatNode = format.doTransform(args) - - val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - val dataTypes = if (timeZoneId.isDefined) { - Seq(original.format.dataType, original.timestamp.dataType, StringType) - } else { - Seq(original.format.dataType, original.timestamp.dataType) - } - - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(substraitExprName, dataTypes)) - - val expressionNodes = new JArrayList[ExpressionNode]() - expressionNodes.add(formatNode) - expressionNodes.add(timestampNode) - if (timeZoneId.isDefined) { - expressionNodes.add(ExpressionBuilder.makeStringLiteral(timeZoneId.get)) - } - - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = { + val timeZoneId = original.timeZoneId.map(timeZoneId => LiteralTransformer(timeZoneId)) + Seq(format, timestamp) ++ timeZoneId } } @@ -158,36 +54,24 @@ case class MonthsBetweenTransformer( date1: ExpressionTransformer, date2: ExpressionTransformer, roundOff: ExpressionTransformer, - timeZoneId: Option[String] = None, original: MonthsBetween) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val date1Node = date1.doTransform(args) - val data2Node = date2.doTransform(args) - val roundOffNode = roundOff.doTransform(args) - - val functionMap = args.asInstanceOf[JHashMap[String, JLong]] - val dataTypes = if (timeZoneId.isDefined) { - Seq(original.date1.dataType, original.date2.dataType, original.roundOff.dataType, StringType) - } else { - Seq(original.date1.dataType, original.date2.dataType, original.roundOff.dataType) - } - - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(substraitExprName, dataTypes)) - - val expressionNodes = new JArrayList[ExpressionNode]() - expressionNodes.add(date1Node) - expressionNodes.add(data2Node) - expressionNodes.add(roundOffNode) - if (timeZoneId.isDefined) { - expressionNodes.add(ExpressionBuilder.makeStringLiteral(timeZoneId.get)) - } + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = { + val timeZoneId = original.timeZoneId.map(timeZoneId => LiteralTransformer(timeZoneId)) + Seq(date1, date2, roundOff) ++ timeZoneId + } +} - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) +case class TimestampAddTransformer( + substraitExprName: String, + unit: String, + left: ExpressionTransformer, + right: ExpressionTransformer, + timeZoneId: String, + original: Expression) + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = { + Seq(LiteralTransformer(unit), left, right, LiteralTransformer(timeZoneId)) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/DecimalRoundTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/DecimalRoundTransformer.scala index 60e64cd95739..305d4feb9338 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/DecimalRoundTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/DecimalRoundTransformer.scala @@ -17,24 +17,20 @@ package org.apache.gluten.expression import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types.{DataType, DecimalType} -import com.google.common.collect.Lists - case class DecimalRoundTransformer( substraitExprName: String, child: ExpressionTransformer, original: Round) - extends ExpressionTransformer { + extends BinaryExpressionTransformer { val toScale: Int = original.scale.eval(EmptyRow).asInstanceOf[Int] // Use the same result type for different Spark versions. - val dataType: DataType = original.child.dataType match { + override val dataType: DataType = original.child.dataType match { case decimalType: DecimalType => val p = decimalType.precision val s = decimalType.scale @@ -57,21 +53,6 @@ case class DecimalRoundTransformer( s"Decimal type is expected but received ${original.child.dataType.typeName}.") } - override def doTransform(args: Object): ExpressionNode = { - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.child.dataType), - FunctionConfig.OPT)) - - ExpressionBuilder.makeScalarFunction( - functionId, - Lists.newArrayList[ExpressionNode]( - child.doTransform(args), - ExpressionBuilder.makeIntLiteral(toScale)), - ConverterUtils.getTypeNode(dataType, original.nullable) - ) - } + override def left: ExpressionTransformer = child + override def right: ExpressionTransformer = LiteralTransformer(toScale) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index e22a20e0dc4c..6e1427e2fda9 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -25,9 +25,8 @@ import org.apache.gluten.test.TestStats import org.apache.gluten.utils.DecimalArithmeticUtil import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} +import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero import org.apache.spark.sql.execution.{ScalarSubquery, _} @@ -37,13 +36,8 @@ import org.apache.spark.sql.hive.HiveUDFTransformer import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -trait Transformable extends Expression { +trait Transformable extends Unevaluable { def getTransformer(childrenTransformers: Seq[ExpressionTransformer]): ExpressionTransformer - - override def eval(input: InternalRow): Any = throw new UnsupportedOperationException() - - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = - throw new UnsupportedOperationException() } object ExpressionConverter extends SQLConfHelper with Logging { @@ -172,7 +166,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { case c: CreateArray => val children = c.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) - CreateArrayTransformer(substraitExprName, children, useStringTypeWhenEmpty = true, c) + CreateArrayTransformer(substraitExprName, children, c) case g: GetArrayItem => BackendsApiManager.getSparkPlanExecApiInstance.genGetArrayItemTransformer( substraitExprName, @@ -183,7 +177,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { case c: CreateMap => val children = c.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) - CreateMapTransformer(substraitExprName, children, c.useStringTypeWhenEmpty, c) + CreateMapTransformer(substraitExprName, children, c) case g: GetMapValue => BackendsApiManager.getSparkPlanExecApiInstance.genGetMapValueTransformer( substraitExprName, @@ -225,14 +219,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { val bindReference = BindReferences.bindReference(expr, attributeSeq, allowFailures = false) val b = bindReference.asInstanceOf[BoundReference] - AttributeReferenceTransformer( - a.name, - b.ordinal, - a.dataType, - b.nullable, - a.exprId, - a.qualifier, - a.metadata) + AttributeReferenceTransformer(substraitExprName, a, b) } catch { case e: IllegalStateException => // This situation may need developers to fix, although we just throw the below @@ -241,11 +228,11 @@ object ExpressionConverter extends SQLConfHelper with Logging { s"Failed to bind reference for $expr: ${e.getMessage}") } case b: BoundReference => - BoundReferenceTransformer(b.ordinal, b.dataType, b.nullable) + BoundReferenceTransformer(substraitExprName, b) case l: Literal => LiteralTransformer(l) case d: DateDiff => - DateDiffTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genDateDiffTransformer( substraitExprName, replaceWithExpressionTransformerInternal(d.endDate, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(d.startDate, attributeSeq, expressionsMap), @@ -257,17 +244,23 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(r.child, attributeSeq, expressionsMap), r) case t: ToUnixTimestamp => - BackendsApiManager.getSparkPlanExecApiInstance.genUnixTimestampTransformer( + // The failOnError depends on the config for ANSI. ANSI is not supported currently. + // And timeZoneId is passed to backend config. + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal(t.timeExp, attributeSeq, expressionsMap), - replaceWithExpressionTransformerInternal(t.format, attributeSeq, expressionsMap), + Seq( + replaceWithExpressionTransformerInternal(t.timeExp, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(t.format, attributeSeq, expressionsMap) + ), t ) case u: UnixTimestamp => - BackendsApiManager.getSparkPlanExecApiInstance.genUnixTimestampTransformer( + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal(u.timeExp, attributeSeq, expressionsMap), - replaceWithExpressionTransformerInternal(u.format, attributeSeq, expressionsMap), + Seq( + replaceWithExpressionTransformerInternal(u.timeExp, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(u.format, attributeSeq, expressionsMap) + ), ToUnixTimestamp(u.timeExp, u.format, u.timeZoneId, u.failOnError) ) case t: TruncTimestamp => @@ -284,11 +277,11 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(m.date1, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(m.date2, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(m.roundOff, attributeSeq, expressionsMap), - m.timeZoneId, m ) case i: If => IfTransformer( + substraitExprName, replaceWithExpressionTransformerInternal(i.predicate, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(i.trueValue, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(i.falseValue, attributeSeq, expressionsMap), @@ -296,6 +289,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { ) case cw: CaseWhen => CaseWhenTransformer( + substraitExprName, cw.branches.map { expr => { @@ -318,26 +312,23 @@ object ExpressionConverter extends SQLConfHelper with Logging { s"In list option does not support non-foldable expression, ${i.list.map(_.sql)}") } InTransformer( + substraitExprName, replaceWithExpressionTransformerInternal(i.value, attributeSeq, expressionsMap), - i.list, - i.value.dataType, i) case i: InSet => InSetTransformer( + substraitExprName, replaceWithExpressionTransformerInternal(i.child, attributeSeq, expressionsMap), - i.hset, - i.child.dataType, i) case s: ScalarSubquery => - ScalarSubqueryTransformer(s.plan, s.exprId, s) + ScalarSubqueryTransformer(substraitExprName, s) case c: Cast => // Add trim node, as necessary. val newCast = BackendsApiManager.getSparkPlanExecApiInstance.genCastWithNewChild(c) CastTransformer( + substraitExprName, replaceWithExpressionTransformerInternal(newCast.child, attributeSeq, expressionsMap), - newCast.dataType, - newCast.timeZoneId, newCast) case s: String2TrimExpression => val (srcStr, trimStr) = s match { @@ -345,10 +336,13 @@ object ExpressionConverter extends SQLConfHelper with Logging { case StringTrimLeft(srcStr, trimStr) => (srcStr, trimStr) case StringTrimRight(srcStr, trimStr) => (srcStr, trimStr) } - String2TrimExpressionTransformer( + val children = trimStr + .map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) + .toSeq ++ + Seq(replaceWithExpressionTransformerInternal(srcStr, attributeSeq, expressionsMap)) + GenericExpressionTransformer( substraitExprName, - trimStr.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), - replaceWithExpressionTransformerInternal(srcStr, attributeSeq, expressionsMap), + children, s ) case m: HashExpression[_] => @@ -368,15 +362,14 @@ object ExpressionConverter extends SQLConfHelper with Logging { getStructField.ordinal, getStructField) case getArrayStructFields: GetArrayStructFields => - GetArrayStructFieldsTransformer( + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal( - getArrayStructFields.child, - attributeSeq, - expressionsMap), - getArrayStructFields.ordinal, - getArrayStructFields.numFields, - getArrayStructFields.containsNull, + Seq( + replaceWithExpressionTransformerInternal( + getArrayStructFields.child, + attributeSeq, + expressionsMap), + LiteralTransformer(getArrayStructFields.ordinal)), getArrayStructFields ) case t: StringTranslate => @@ -387,14 +380,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(t.replaceExpr, attributeSeq, expressionsMap), t ) - case l: StringLocate => - BackendsApiManager.getSparkPlanExecApiInstance.genStringLocateTransformer( - substraitExprName, - replaceWithExpressionTransformerInternal(l.first, attributeSeq, expressionsMap), - replaceWithExpressionTransformerInternal(l.second, attributeSeq, expressionsMap), - replaceWithExpressionTransformerInternal(l.third, attributeSeq, expressionsMap), - l - ) case s: StringSplit => BackendsApiManager.getSparkPlanExecApiInstance.genStringSplitTransformer( substraitExprName, @@ -414,23 +399,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { ), r ) - case md5: Md5 => - BackendsApiManager.getSparkPlanExecApiInstance.genMd5Transformer( - substraitExprName, - replaceWithExpressionTransformerInternal(md5.child, attributeSeq, expressionsMap), - md5) - case sha1: Sha1 => - BackendsApiManager.getSparkPlanExecApiInstance.genSha1Transformer( - substraitExprName, - replaceWithExpressionTransformerInternal(sha1.child, attributeSeq, expressionsMap), - sha1) - case sha2: Sha2 => - BackendsApiManager.getSparkPlanExecApiInstance.genSha2Transformer( - substraitExprName, - replaceWithExpressionTransformerInternal(sha2.left, attributeSeq, expressionsMap), - replaceWithExpressionTransformerInternal(sha2.right, attributeSeq, expressionsMap), - sha2 - ) case size: Size => if (size.legacySizeOfNull != SQLConf.get.legacySizeOfNull) { throw new GlutenNotSupportException( @@ -449,12 +417,11 @@ object ExpressionConverter extends SQLConfHelper with Logging { namedStruct, attributeSeq) case namedLambdaVariable: NamedLambdaVariable => - NamedLambdaVariableTransformer( + // namedlambdavariable('acc')-> + GenericExpressionTransformer( substraitExprName, - name = namedLambdaVariable.name, - dataType = namedLambdaVariable.dataType, - nullable = namedLambdaVariable.nullable, - exprId = namedLambdaVariable.exprId + LiteralTransformer(namedLambdaVariable.name), + namedLambdaVariable ) case lambdaFunction: LambdaFunction => LambdaFunctionTransformer( @@ -472,25 +439,33 @@ object ExpressionConverter extends SQLConfHelper with Logging { j.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)) JsonTupleExpressionTransformer(substraitExprName, children, j) case l: Like => - LikeTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genLikeTransformer( substraitExprName, replaceWithExpressionTransformerInternal(l.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(l.right, attributeSeq, expressionsMap), l ) case m: MakeDecimal => - MakeDecimalTransformer( + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal(m.child, attributeSeq, expressionsMap), - m) + Seq( + replaceWithExpressionTransformerInternal(m.child, attributeSeq, expressionsMap), + LiteralTransformer(m.nullOnOverflow)), + m + ) case rand: Rand => BackendsApiManager.getSparkPlanExecApiInstance.genRandTransformer( substraitExprName, replaceWithExpressionTransformerInternal(rand.child, attributeSeq, expressionsMap), rand) - case _: NormalizeNaNAndZero | _: PromotePrecision => + case _: NormalizeNaNAndZero | _: PromotePrecision | _: TaggingExpression => ChildTransformer( - replaceWithExpressionTransformerInternal(expr.children.head, attributeSeq, expressionsMap) + substraitExprName, + replaceWithExpressionTransformerInternal( + expr.children.head, + attributeSeq, + expressionsMap), + expr ) case _: GetDateField | _: GetTimeField => ExtractDateTransformer( @@ -524,7 +499,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { CheckOverflowTransformer( substraitExprName, replaceWithExpressionTransformerInternal(c.child, attributeSeq, expressionsMap), - c.child.dataType, c) case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b) => DecimalArithmeticUtil.checkAllowDecimalArithmetic() @@ -565,12 +539,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(add.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(add.right, attributeSeq, expressionsMap), extract.get.last, - add.dataType, - add.nullable - ) - case e: TaggingExpression => - ChildTransformer( - replaceWithExpressionTransformerInternal(e.child, attributeSeq, expressionsMap) + add ) case e: Transformable => val childrenTransformers = @@ -614,7 +583,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { ) case tryEval: TryEval => // This is a placeholder to handle try_eval(other expressions). - BackendsApiManager.getSparkPlanExecApiInstance.genTryAddTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genTryEvalTransformer( substraitExprName, replaceWithExpressionTransformerInternal(tryEval.child, attributeSeq, expressionsMap), tryEval diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala index 6b65878627c5..ebb9db3e824f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala @@ -16,17 +16,74 @@ */ package org.apache.gluten.expression -import org.apache.gluten.substrait.expression.ExpressionNode +import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} -import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} import org.apache.spark.sql.types.DataType -trait ExpressionTransformer { - def doTransform(args: java.lang.Object): ExpressionNode - def dataType: DataType -} +import scala.collection.JavaConverters._ + +// ==== Expression transformer basic interface start ==== -trait ExpressionTransformerWithOrigin extends ExpressionTransformer { +trait ExpressionTransformer { + def substraitExprName: String + def children: Seq[ExpressionTransformer] def original: Expression def dataType: DataType = original.dataType + def nullable: Boolean = original.nullable + + def doTransform(args: java.lang.Object): ExpressionNode = { + val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] + // TODO: the funcName seems can be simplified to `substraitExprName` + val funcName: String = + ConverterUtils.makeFuncName(substraitExprName, original.children.map(_.dataType)) + val functionId = ExpressionBuilder.newScalarFunction(functionMap, funcName) + val childNodes = children.map(_.doTransform(args)).asJava + val typeNode = ConverterUtils.getTypeNode(dataType, nullable) + ExpressionBuilder.makeScalarFunction(functionId, childNodes, typeNode) + } +} + +trait LeafExpressionTransformer extends ExpressionTransformer { + final override def children: Seq[ExpressionTransformer] = Nil +} + +trait UnaryExpressionTransformer extends ExpressionTransformer { + def child: ExpressionTransformer + final override def children: Seq[ExpressionTransformer] = child :: Nil +} + +trait BinaryExpressionTransformer extends ExpressionTransformer { + def left: ExpressionTransformer + def right: ExpressionTransformer + final override def children: Seq[ExpressionTransformer] = left :: right :: Nil +} + +// ==== Expression transformer basic interface end ==== + +case class GenericExpressionTransformer( + substraitExprName: String, + children: Seq[ExpressionTransformer], + original: Expression) + extends ExpressionTransformer + +object GenericExpressionTransformer { + def apply( + substraitExprName: String, + child: ExpressionTransformer, + original: Expression): GenericExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, child :: Nil, original) + } +} + +case class LiteralTransformer(original: Literal) extends LeafExpressionTransformer { + override def substraitExprName: String = "literal" + override def doTransform(args: java.lang.Object): ExpressionNode = { + ExpressionBuilder.makeLiteral(original.value, original.dataType, original.nullable) + } +} +object LiteralTransformer { + def apply(v: Any): LiteralTransformer = { + LiteralTransformer(Literal(v)) + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala deleted file mode 100644 index 8faf4965fb4d..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/GenericExpressionTransformer.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} - -import org.apache.spark.sql.catalyst.expressions._ - -import com.google.common.collect.Lists - -case class GenericExpressionTransformer( - substraitExprName: String, - children: Seq[ExpressionTransformer], - original: Expression) - extends ExpressionTransformerWithOrigin { - override def doTransform(args: Object): ExpressionNode = { - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT)) - - val exprNodes = Lists.newArrayList[ExpressionNode]() - children.foreach(expr => exprNodes.add(expr.doTransform(args))) - - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, exprNodes, typeNode) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala deleted file mode 100644 index 28f2dda01e61..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/HashExpressionTransformer.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} - -import org.apache.spark.sql.catalyst.expressions._ - -case class HashExpressionTransformer( - substraitExprName: String, - exps: Seq[ExpressionTransformer], - original: Expression) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val nodes = new java.util.ArrayList[ExpressionNode]() - exps.foreach( - expression => { - nodes.add(expression.doTransform(args)) - }) - val childrenTypes = original.children.map(child => child.dataType) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionName = - ConverterUtils.makeFuncName(substraitExprName, childrenTypes, FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, nodes, typeNode) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala index e8ff3d360a8d..25e3e12a53de 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/JsonTupleExpressionTransformer.scala @@ -28,7 +28,7 @@ case class JsonTupleExpressionTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], original: Expression) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { override def doTransform(args: Object): ExpressionNode = { val jsonExpr = children.head diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala index ce6d13a95181..9e7285ac3a17 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/LambdaFunctionTransformer.scala @@ -17,7 +17,7 @@ package org.apache.gluten.expression import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} +import org.apache.gluten.substrait.expression.ExpressionNode import org.apache.spark.sql.catalyst.expressions.LambdaFunction @@ -25,27 +25,15 @@ case class LambdaFunctionTransformer( substraitExprName: String, function: ExpressionTransformer, arguments: Seq[ExpressionTransformer], - hidden: Boolean = false, original: LambdaFunction) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { + override def children: Seq[ExpressionTransformer] = function +: arguments override def doTransform(args: Object): ExpressionNode = { // Need to fallback when hidden be true as it's not supported in Velox - if (hidden) { + if (original.hidden) { throw new GlutenNotSupportException(s"Unsupported LambdaFunction with hidden be true.") } - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.dataType), - ConverterUtils.FunctionConfig.OPT)) - val expressionNodes = new java.util.ArrayList[ExpressionNode] - expressionNodes.add(function.doTransform(args)) - arguments.foreach(argument => expressionNodes.add(argument.doTransform(args))) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) + super.doTransform(args) } - } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala deleted file mode 100644 index 8fb9943d6398..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/LiteralTransformer.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} - -import org.apache.spark.sql.catalyst.expressions._ - -case class LiteralTransformer(original: Literal) extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - ExpressionBuilder.makeLiteral(original.value, original.dataType, original.nullable) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala index c09afaebc35a..fe715979b1a7 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/MapExpressionTransformer.scala @@ -18,53 +18,35 @@ package org.apache.gluten.expression import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} +import org.apache.gluten.substrait.expression.ExpressionNode import org.apache.spark.sql.catalyst.expressions._ -import com.google.common.collect.Lists - case class CreateMapTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], - useStringTypeWhenEmpty: Boolean, original: CreateMap) - extends ExpressionTransformerWithOrigin { + extends ExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { // If children is empty, // transformation is only supported when useStringTypeWhenEmpty is false // because ClickHouse and Velox currently doesn't support this config. - if (children.isEmpty && useStringTypeWhenEmpty) { + if (children.isEmpty && original.useStringTypeWhenEmpty) { throw new GlutenNotSupportException(s"$original not supported yet.") } - val childNodes = new java.util.ArrayList[ExpressionNode]() - children.foreach( - child => { - val childNode = child.doTransform(args) - childNodes.add(childNode) - }) - - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, childNodes, typeNode) + super.doTransform(args) } } case class GetMapValueTransformer( substraitExprName: String, - child: ExpressionTransformer, - key: ExpressionTransformer, + left: ExpressionTransformer, + right: ExpressionTransformer, failOnError: Boolean, original: GetMapValue) - extends ExpressionTransformerWithOrigin { + extends BinaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { if (BackendsApiManager.getSettings.alwaysFailOnMapExpression()) { @@ -75,17 +57,6 @@ case class GetMapValueTransformer( throw new GlutenNotSupportException(s"$original not supported yet.") } - val childNode = child.doTransform(args) - val keyNode = key.doTransform(args) - - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.child.dataType, original.key.dataType), - FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val exprNodes = Lists.newArrayList(childNode, keyNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, exprNodes, typeNode) + super.doTransform(args) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala index 2af4a5fa2558..f4c703d88ef0 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/NamedExpressionsTransformer.scala @@ -16,67 +16,29 @@ */ package org.apache.gluten.expression -import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.types._ - -import com.google.common.collect.Lists case class AliasTransformer( substraitExprName: String, child: ExpressionTransformer, original: Expression) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val childNode = child.doTransform(args) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.REQ)) - val expressionNodes = Lists.newArrayList(childNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) - } -} + extends UnaryExpressionTransformer {} -case class NamedLambdaVariableTransformer( +case class AttributeReferenceTransformer( substraitExprName: String, - name: String, - dataType: DataType, - nullable: Boolean, - exprId: ExprId) - extends ExpressionTransformer { - override def doTransform(args: Object): ExpressionNode = { - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val namedLambdaVarFunctionName = - ConverterUtils.makeFuncName(substraitExprName, Seq(dataType), FunctionConfig.OPT) - val arrayAggFunctionId = - ExpressionBuilder.newScalarFunction(functionMap, namedLambdaVarFunctionName) - val exprNodes = Lists.newArrayList( - ExpressionBuilder.makeLiteral(name, StringType, false).asInstanceOf[ExpressionNode]) - val typeNode = ConverterUtils.getTypeNode(dataType, nullable) - // namedlambdavariable('acc')-> - ExpressionBuilder.makeScalarFunction(arrayAggFunctionId, exprNodes, typeNode) + original: AttributeReference, + bound: BoundReference) + extends LeafExpressionTransformer { + override def doTransform(args: java.lang.Object): ExpressionNode = { + ExpressionBuilder.makeSelection(bound.ordinal.asInstanceOf[java.lang.Integer]) } } -case class AttributeReferenceTransformer( - name: String, - ordinal: Int, - dataType: DataType, - nullable: Boolean = true, - exprId: ExprId, - qualifier: Seq[String], - metadata: Metadata = Metadata.empty) - extends ExpressionTransformer { - +case class BoundReferenceTransformer(substraitExprName: String, original: BoundReference) + extends LeafExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { - ExpressionBuilder.makeSelection(ordinal.asInstanceOf[java.lang.Integer]) + ExpressionBuilder.makeSelection(original.ordinal.asInstanceOf[java.lang.Integer]) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala index 7d34466e5044..d13c61d64af3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala @@ -16,39 +16,33 @@ */ package org.apache.gluten.expression -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types._ -import com.google.common.collect.Lists - import scala.collection.JavaConverters._ -case class InTransformer( - value: ExpressionTransformer, - list: Seq[Expression], - valueType: DataType, - original: Expression) - extends ExpressionTransformerWithOrigin { +case class InTransformer(substraitExprName: String, child: ExpressionTransformer, original: In) + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { - assert(list.forall(_.foldable)) + assert(original.list.forall(_.foldable)) // Stores the values in a List Literal. - val values: Set[Any] = list.map(_.eval()).toSet - InExpressionTransformer.toTransformer(value.doTransform(args), values, valueType) + val values: Set[Any] = original.list.map(_.eval()).toSet + InExpressionTransformer.toTransformer(child.doTransform(args), values, child.dataType) } } case class InSetTransformer( - value: ExpressionTransformer, - hset: Set[Any], - valueType: DataType, - original: Expression) - extends ExpressionTransformerWithOrigin { + substraitExprName: String, + child: ExpressionTransformer, + original: InSet) + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { - InExpressionTransformer.toTransformer(value.doTransform(args), hset, valueType) + InExpressionTransformer.toTransformer( + child.doTransform(args), + original.hset, + original.child.dataType) } } @@ -69,60 +63,12 @@ object InExpressionTransformer { } } -case class LikeTransformer( - substraitExprName: String, - left: ExpressionTransformer, - right: ExpressionTransformer, - original: Expression) - extends ExpressionTransformerWithOrigin { - override def doTransform(args: java.lang.Object): ExpressionNode = { - val leftNode = left.doTransform(args) - val rightNode = right.doTransform(args) - val escapeCharNode = ExpressionBuilder.makeLiteral( - original.asInstanceOf[Like].escapeChar.toString, - StringType, - false) - - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT)) - - // CH backend does not support escapeChar, so skip it here. - val expressionNodes = - BackendsApiManager.getTransformerApiInstance.createLikeParamList( - leftNode, - rightNode, - escapeCharNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes.toList.asJava, typeNode) - } -} - case class DecimalArithmeticExpressionTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, resultType: DecimalType, original: Expression) - extends ExpressionTransformerWithOrigin { + extends BinaryExpressionTransformer { override def dataType: DataType = resultType - override def doTransform(args: java.lang.Object): ExpressionNode = { - val leftNode = left.doTransform(args) - val rightNode = right.doTransform(args) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT)) - - val expressionNodes = Lists.newArrayList(leftNode, rightNode) - val typeNode = ConverterUtils.getTypeNode(resultType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) - } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala index 4f5a43d47646..0accf9ffd0f9 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala @@ -19,10 +19,10 @@ package org.apache.gluten.expression import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.{BaseSubqueryExec, ScalarSubquery} +import org.apache.spark.sql.execution.ScalarSubquery -case class ScalarSubqueryTransformer(plan: BaseSubqueryExec, exprId: ExprId, query: ScalarSubquery) - extends ExpressionTransformerWithOrigin { +case class ScalarSubqueryTransformer(substraitExprName: String, query: ScalarSubquery) + extends LeafExpressionTransformer { override def original: Expression = query override def doTransform(args: java.lang.Object): ExpressionNode = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala deleted file mode 100644 index b31d66b68e0a..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/StringExpressionTransformer.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression._ - -import org.apache.spark.sql.catalyst.expressions._ - -import com.google.common.collect.Lists - -case class String2TrimExpressionTransformer( - substraitExprName: String, - trimStr: Option[ExpressionTransformer], - srcStr: ExpressionTransformer, - original: Expression) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val trimStrNode = trimStr.map(_.doTransform(args)) - val srcStrNode = srcStr.doTransform(args) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionName = - ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.REQ) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val expressNodes = Lists.newArrayList[ExpressionNode]() - trimStrNode.foreach(expressNodes.add) - expressNodes.add(srcStrNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressNodes, typeNode) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala deleted file mode 100644 index 616971b6d15f..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/StructExpressionTransformer.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, StructLiteralNode} - -import org.apache.spark.sql.catalyst.expressions.GetStructField -import org.apache.spark.sql.types.IntegerType - -import com.google.common.collect.Lists - -case class GetStructFieldTransformer( - substraitExprName: String, - childTransformer: ExpressionTransformer, - ordinal: Int, - original: GetStructField) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val childNode = childTransformer.doTransform(args) - childNode match { - case node: StructLiteralNode => - return node.getFieldLiteral(ordinal) - case _ => - } - - val ordinalNode = ExpressionBuilder.makeLiteral(ordinal, IntegerType, false) - val exprNodes = Lists.newArrayList(childNode, ordinalNode) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val fieldDataType = original.dataType - val functionName = ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.child.dataType, fieldDataType), - FunctionConfig.OPT) - val functionId = ExpressionBuilder.newScalarFunction(functionMap, functionName) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, exprNodes, typeNode) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/TimestampAddTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/TimestampAddTransformer.scala deleted file mode 100644 index acede4523846..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/TimestampAddTransformer.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.expression - -import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} - -import org.apache.spark.sql.types.DataType - -import com.google.common.collect.Lists - -case class TimestampAddTransformer( - substraitExprName: String, - unit: String, - left: ExpressionTransformer, - right: ExpressionTransformer, - timeZoneId: String, - dataType: DataType, - nullable: Boolean) - extends ExpressionTransformer { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val leftNode = left.doTransform(args) - val rightNode = right.doTransform(args) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(substraitExprName, Seq(), FunctionConfig.REQ) - ) - - val expressionNodes = Lists.newArrayList( - ExpressionBuilder.makeStringLiteral(unit), - leftNode, - rightNode, - ExpressionBuilder.makeStringLiteral(timeZoneId)) - val outputType = ConverterUtils.getTypeNode(dataType, nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, outputType) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala index d0ac19b4a9e3..27f8395254fd 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala @@ -18,30 +18,29 @@ package org.apache.gluten.expression import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.substrait.`type`.ListNode import org.apache.gluten.substrait.`type`.MapNode -import org.apache.gluten.substrait.expression.{BooleanLiteralNode, ExpressionBuilder, ExpressionNode, IntLiteralNode} +import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, StructLiteralNode} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types._ import com.google.common.collect.Lists -case class ChildTransformer(child: ExpressionTransformer) extends ExpressionTransformer { +case class ChildTransformer( + substraitExprName: String, + child: ExpressionTransformer, + original: Expression) + extends UnaryExpressionTransformer { + override def dataType: DataType = child.dataType + override def doTransform(args: java.lang.Object): ExpressionNode = { child.doTransform(args) } - override def dataType: DataType = child.dataType } -case class CastTransformer( - child: ExpressionTransformer, - dataType: DataType, - timeZoneId: Option[String], - original: Cast) - extends ExpressionTransformer { - +case class CastTransformer(substraitExprName: String, child: ExpressionTransformer, original: Cast) + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable) ExpressionBuilder.makeCast(typeNode, child.doTransform(args), original.ansiEnabled) @@ -52,7 +51,7 @@ case class ExplodeTransformer( substraitExprName: String, child: ExpressionTransformer, original: Explode) - extends ExpressionTransformerWithOrigin { + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { val childNode: ExpressionNode = child.doTransform(args) @@ -75,123 +74,23 @@ case class ExplodeTransformer( } } -case class PosExplodeTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: PosExplode, - attributeSeq: Seq[Attribute]) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val childNode: ExpressionNode = child.doTransform(args) - - // sequence(1, size(array_or_map)) - val startExpr = new Literal(1, IntegerType) - val stopExpr = new Size(Size(original.child, false)) - val stepExpr = new Literal(1, IntegerType) - val sequenceExpr = new Sequence(startExpr, stopExpr, stepExpr) - val sequenceExprNode = ExpressionConverter - .replaceWithExpressionTransformer(sequenceExpr, attributeSeq) - .doTransform(args) - - val funcMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - - val mapFromArraysFuncId = ExpressionBuilder.newScalarFunction( - funcMap, - ConverterUtils.makeFuncName( - ExpressionNames.MAP_FROM_ARRAYS, - Seq(sequenceExpr.dataType, original.child.dataType), - FunctionConfig.OPT)) - - val keyType = IntegerType - val (valType, valContainsNull) = original.child.dataType match { - case a: ArrayType => (a.elementType, a.containsNull) - case _ => - throw new GlutenNotSupportException( - s"posexplode(${original.child.dataType}) not supported yet.") - } - val outputType = MapType(keyType, valType, valContainsNull) - val mapFromArraysExprNode = ExpressionBuilder.makeScalarFunction( - mapFromArraysFuncId, - Lists.newArrayList(sequenceExprNode, childNode), - ConverterUtils.getTypeNode(outputType, original.child.nullable)) - - // posexplode(map_from_arrays(sequence(1, size(array_or_map)), array_or_map)) - val funcId = ExpressionBuilder.newScalarFunction( - funcMap, - ConverterUtils.makeFuncName(ExpressionNames.POSEXPLODE, Seq(outputType), FunctionConfig.OPT)) - - val childType = original.child.dataType - childType match { - case a: ArrayType => - // Output pos, col when input is array - val structType = StructType( - Array( - StructField("pos", IntegerType, false), - StructField("col", a.elementType, a.containsNull))) - ExpressionBuilder.makeScalarFunction( - funcId, - Lists.newArrayList(mapFromArraysExprNode), - ConverterUtils.getTypeNode(structType, false)) - case m: MapType => - // Output pos, key, value when input is map - val structType = StructType( - Array( - StructField("pos", IntegerType, false), - StructField("key", m.keyType, false), - StructField("value", m.valueType, m.valueContainsNull))) - ExpressionBuilder.makeScalarFunction( - funcId, - Lists.newArrayList(mapFromArraysExprNode), - ConverterUtils.getTypeNode(structType, false)) - case _ => - throw new GlutenNotSupportException(s"posexplode($childType) not supported yet.") - } - } -} - case class CheckOverflowTransformer( substraitExprName: String, child: ExpressionTransformer, - childResultType: DataType, original: CheckOverflow) - extends ExpressionTransformerWithOrigin { - + extends UnaryExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { BackendsApiManager.getTransformerApiInstance.createCheckOverflowExprNode( args, substraitExprName, child.doTransform(args), - childResultType, + original.child.dataType, original.dataType, original.nullable, original.nullOnOverflow) } } -case class MakeDecimalTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: MakeDecimal) - extends ExpressionTransformerWithOrigin { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - val childNode = child.doTransform(args) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.dataType, BooleanType), - FunctionConfig.OPT)) - - val expressionNodes = - Lists.newArrayList(childNode, new BooleanLiteralNode(original.nullOnOverflow)) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) - } -} - /** * User can specify a seed for this function. If lacked, spark will generate a random number as * seed. We also need to pass a unique partitionIndex provided by framework to native library for @@ -203,43 +102,32 @@ case class RandTransformer( substraitExprName: String, explicitSeed: ExpressionTransformer, original: Rand) - extends ExpressionTransformerWithOrigin { + extends LeafExpressionTransformer { override def doTransform(args: java.lang.Object): ExpressionNode = { if (!original.hideSeed) { // TODO: for user-specified seed, we need to pass partition index to native engine. throw new GlutenNotSupportException("User-specified seed is not supported.") } - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(substraitExprName, Seq(original.child.dataType))) - val inputNodes = Lists.newArrayList[ExpressionNode]() - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, inputNodes, typeNode) + super.doTransform(args) } } -case class GetArrayStructFieldsTransformer( +case class GetStructFieldTransformer( substraitExprName: String, child: ExpressionTransformer, - ordinal: Int, - numFields: Int, - containsNull: Boolean, - original: GetArrayStructFields) - extends ExpressionTransformerWithOrigin { + original: GetStructField) + extends BinaryExpressionTransformer { + override def left: ExpressionTransformer = child + override def right: ExpressionTransformer = LiteralTransformer(original.ordinal) override def doTransform(args: java.lang.Object): ExpressionNode = { - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - Seq(original.child.dataType, IntegerType), - FunctionConfig.OPT)) - val inputNodes = - Lists.newArrayList(child.doTransform(args), new IntLiteralNode(ordinal)) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, inputNodes, typeNode) + val childNode = child.doTransform(args) + childNode match { + case node: StructLiteralNode => + node.getFieldLiteral(original.ordinal) + case _ => + super.doTransform(args) + } } } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala index c27159cebdda..f6ff0ff45564 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/CustomerExpressionTransformer.scala @@ -17,38 +17,16 @@ package org.apache.spark.sql.extension import org.apache.gluten.expression._ -import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.extension.ExpressionExtensionTrait -import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} -import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import com.google.common.collect.Lists - case class CustomAddExpressionTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, original: Expression) - extends ExpressionTransformerWithOrigin - with Logging { - override def doTransform(args: java.lang.Object): ExpressionNode = { - val leftNode = left.doTransform(args) - val rightNode = right.doTransform(args) - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName( - substraitExprName, - original.children.map(_.dataType), - FunctionConfig.OPT)) - - val expressionNodes = Lists.newArrayList(leftNode, rightNode) - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) - } -} + extends BinaryExpressionTransformer case class CustomerExpressionTransformer() extends ExpressionExtensionTrait { @@ -65,7 +43,7 @@ case class CustomerExpressionTransformer() extends ExpressionExtensionTrait { expr: Expression, attributeSeq: Seq[Attribute]): ExpressionTransformer = expr match { case custom: CustomAdd => - new CustomAddExpressionTransformer( + CustomAddExpressionTransformer( substraitExprName, ExpressionConverter.replaceWithExpressionTransformer(custom.left, attributeSeq), ExpressionConverter.replaceWithExpressionTransformer(custom.right, attributeSeq), From 3bef31298e53c6c23cdb7e0fc09f60d655f5a505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Tue, 21 May 2024 17:01:54 +0800 Subject: [PATCH 120/402] [VL] Refactor data filter in scan transformer (#5812) [VL] Refactor data filter in scan transformer. --- .../main/scala/org/apache/gluten/sql/shims/SparkShims.scala | 2 ++ .../org/apache/gluten/sql/shims/spark32/Spark32Shims.scala | 2 ++ .../org/apache/gluten/sql/shims/spark33/Spark33Shims.scala | 2 ++ .../org/apache/gluten/sql/shims/spark34/Spark34Shims.scala | 4 ++++ .../apache/spark/sql/execution/FileSourceScanExecShim.scala | 5 ++++- .../org/apache/gluten/sql/shims/spark35/Spark35Shims.scala | 5 +++++ .../apache/spark/sql/execution/FileSourceScanExecShim.scala | 4 ++-- 7 files changed, 21 insertions(+), 3 deletions(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala index fd8cd24c393e..d6acc8c27b29 100644 --- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala @@ -193,6 +193,8 @@ trait SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] + def isRowIndexMetadataColumn(name: String): Boolean + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala index 97251a7ef386..29fddc697b07 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala @@ -189,6 +189,8 @@ class Spark32Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files + def isRowIndexMetadataColumn(name: String): Boolean = false + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala index 58dc0a00cec4..7c6ce644dc74 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala @@ -273,6 +273,8 @@ class Spark33Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files + def isRowIndexMetadataColumn(name: String): Boolean = false + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 6d70d67f313c..4ab307e8568f 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -327,6 +327,10 @@ class Spark34Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files + def isRowIndexMetadataColumn(name: String): Boolean = { + name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + } + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 15455d51c7a9..33df953f32c8 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.metrics.GlutenTimeMetric +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} @@ -56,7 +57,9 @@ abstract class FileSourceScanExecShim( case FileSourceGeneratedMetadataAttribute(attr) => attr } - def dataFiltersInScan: Seq[Expression] = dataFilters + def dataFiltersInScan: Seq[Expression] = dataFilters.filterNot(_.references.exists { + attr => SparkShimLoader.getSparkShims.isRowIndexMetadataColumn(attr.name) + }) def hasUnsupportedColumns: Boolean = { val metadataColumnsNames = metadataColumns.map(_.name) diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 00f9d62fd211..ef1cea865d49 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -42,6 +42,7 @@ import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Sca import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil @@ -355,6 +356,10 @@ class Spark35Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files.map(_.fileStatus) + def isRowIndexMetadataColumn(name: String): Boolean = { + name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + } + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 6295bcbc46d4..dccf1bbced1d 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.metrics.GlutenTimeMetric +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} @@ -59,8 +60,7 @@ abstract class FileSourceScanExecShim( protected lazy val driverMetricsAlias = driverMetrics def dataFiltersInScan: Seq[Expression] = dataFilters.filterNot(_.references.exists { - case FileSourceMetadataAttribute(_) => true - case _ => false + attr => SparkShimLoader.getSparkShims.isRowIndexMetadataColumn(attr.name) }) def hasUnsupportedColumns: Boolean = { From 768301a5c45a5b4b2ee1802361cbeed1aa75a2c4 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 21 May 2024 22:08:57 +0800 Subject: [PATCH 121/402] [CORE] Remove duplicate pipeline metrics measurement (#5821) --- .../org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 5a165116c647..f1fbf3648bb2 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -45,7 +45,6 @@ import java.lang.{Long => JLong} import java.nio.charset.StandardCharsets import java.time.ZoneOffset import java.util.{ArrayList => JArrayList, HashMap => JHashMap, Map => JMap} -import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ @@ -161,7 +160,6 @@ class VeloxIteratorApi extends IteratorApi with Logging { inputPartition.isInstanceOf[GlutenPartition], "Velox backend only accept GlutenPartition.") - val beforeBuild = System.nanoTime() val columnarNativeIterators = new JArrayList[GeneralInIterator](inputIterators.map { iter => new ColumnarBatchInIterator(iter.asJava) @@ -177,7 +175,6 @@ class VeloxIteratorApi extends IteratorApi with Logging { splitInfoByteArray, columnarNativeIterators, partitionIndex) - pipelineTime += TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beforeBuild) Iterators .wrap(resIter.asScala) From 1c6c7fcc2eb59831e21ae7b24500dae883799bd6 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 22 May 2024 09:42:17 +0800 Subject: [PATCH 122/402] [VL] Add config for memory pool init capacity to reduce arbitration times (#5815) --- .../apache/gluten/execution/VeloxTPCHSuite.scala | 1 + cpp/velox/config/VeloxConfig.h | 3 +++ cpp/velox/memory/VeloxMemoryManager.cc | 12 +++++++++--- cpp/velox/tests/MemoryManagerTest.cc | 13 ++++++++----- .../main/scala/org/apache/gluten/GlutenConfig.scala | 7 +++++++ 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 47f0b2c69147..17f27a407e52 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -42,6 +42,7 @@ abstract class VeloxTPCHTableSupport extends VeloxWholeStageTransformerSuite { .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") .set("spark.sql.files.maxPartitionBytes", "1g") .set("spark.sql.shuffle.partitions", "1") + .set("spark.gluten.sql.columnar.backend.velox.memInitCapacity", "1m") .set("spark.memory.offHeap.size", "2g") .set("spark.unsafe.exceptionOnMemoryLeak", "true") .set("spark.sql.autoBroadcastJoinThreshold", "-1") diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h index a3112f83ee35..f57f1293e22e 100644 --- a/cpp/velox/config/VeloxConfig.h +++ b/cpp/velox/config/VeloxConfig.h @@ -70,6 +70,9 @@ const bool kEnableSystemExceptionStacktraceDefault = true; const std::string kMemoryUseHugePages = "spark.gluten.sql.columnar.backend.velox.memoryUseHugePages"; const bool kMemoryUseHugePagesDefault = false; +const std::string kVeloxMemInitCapacity = "spark.gluten.sql.columnar.backend.velox.memInitCapacity"; +const uint64_t kVeloxMemInitCapacityDefault = 8 << 20; + const std::string kHiveConnectorId = "test-hive"; const std::string kVeloxCacheEnabled = "spark.gluten.sql.columnar.backend.velox.cacheEnabled"; diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 0584780ad5ab..b7bd3a9f9a58 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -42,8 +42,12 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { } uint64_t growCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { - VELOX_CHECK_EQ(targetBytes, 0, "Gluten has set MemoryManagerOptions.memoryPoolInitCapacity to 0") - return 0; + std::lock_guard l(mutex_); + listener_->allocationChanged(targetBytes); + if (!pool->grow(targetBytes, 0)) { + VELOX_FAIL("Failed to grow root pool's capacity for {}", velox::succinctBytes(targetBytes)); + } + return targetBytes; } uint64_t shrinkCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { @@ -160,6 +164,8 @@ VeloxMemoryManager::VeloxMemoryManager( : MemoryManager(), name_(name), listener_(std::move(listener)) { auto reservationBlockSize = VeloxBackend::get()->getBackendConf()->get( kMemoryReservationBlockSize, kMemoryReservationBlockSizeDefault); + auto memInitCapacity = + VeloxBackend::get()->getBackendConf()->get(kVeloxMemInitCapacity, kVeloxMemInitCapacityDefault); blockListener_ = std::make_unique(listener_.get(), reservationBlockSize); listenableAlloc_ = std::make_unique(allocator.get(), blockListener_.get()); arrowPool_ = std::make_unique(listenableAlloc_.get()); @@ -173,7 +179,7 @@ VeloxMemoryManager::VeloxMemoryManager( .coreOnAllocationFailureEnabled = false, .allocatorCapacity = velox::memory::kMaxMemory, .arbitratorKind = afr.getKind(), - .memoryPoolInitCapacity = 0, + .memoryPoolInitCapacity = memInitCapacity, .memoryPoolTransferCapacity = reservationBlockSize, .memoryReclaimWaitMs = 0}; veloxMemoryManager_ = std::make_unique(mmOptions); diff --git a/cpp/velox/tests/MemoryManagerTest.cc b/cpp/velox/tests/MemoryManagerTest.cc index f256db1b2bb5..400beafcc1f8 100644 --- a/cpp/velox/tests/MemoryManagerTest.cc +++ b/cpp/velox/tests/MemoryManagerTest.cc @@ -17,7 +17,7 @@ #include "benchmarks/common/BenchmarkUtils.h" #include "compute/VeloxBackend.h" -#include "config/GlutenConfig.h" +#include "config/VeloxConfig.h" #include "memory/VeloxMemoryManager.h" #include "velox/common/base/tests/GTestUtils.h" @@ -48,7 +48,8 @@ class MemoryManagerTest : public ::testing::Test { protected: static void SetUpTestCase() { std::unordered_map conf = { - {kMemoryReservationBlockSize, std::to_string(kMemoryReservationBlockSizeDefault)}}; + {kMemoryReservationBlockSize, std::to_string(kMemoryReservationBlockSizeDefault)}, + {kVeloxMemInitCapacity, std::to_string(kVeloxMemInitCapacityDefault)}}; initVeloxBackend(conf); } @@ -93,6 +94,8 @@ TEST_F(MemoryManagerTest, memoryPoolWithBlockReseravtion) { } TEST_F(MemoryManagerTest, memoryAllocatorWithBlockReservation) { + auto initBytes = listener_->currentBytes(); + std::vector allocations; std::vector sizes{ kMemoryReservationBlockSizeDefault - 1 * kMB, kMemoryReservationBlockSizeDefault - 2 * kMB}; @@ -105,7 +108,7 @@ TEST_F(MemoryManagerTest, memoryAllocatorWithBlockReservation) { EXPECT_EQ(allocator_->getBytes(), currentBytes + size); EXPECT_EQ(allocator_->peakBytes(), allocator_->getBytes()); - EXPECT_EQ(listener_->currentBytes(), (i + 1) * kMemoryReservationBlockSizeDefault); + EXPECT_EQ(listener_->currentBytes(), (i + 1) * kMemoryReservationBlockSizeDefault + initBytes); EXPECT_EQ(listener_->peakBytes(), listener_->currentBytes()); } @@ -114,14 +117,14 @@ TEST_F(MemoryManagerTest, memoryAllocatorWithBlockReservation) { allocations.pop_back(); allocator_->free(allocation.buffer, allocation.size); EXPECT_EQ(allocator_->getBytes(), currentBytes - allocation.size); - EXPECT_EQ(listener_->currentBytes(), kMemoryReservationBlockSizeDefault); + EXPECT_EQ(listener_->currentBytes(), kMemoryReservationBlockSizeDefault + initBytes); currentBytes = allocator_->getBytes(); allocation = allocations.back(); allocations.pop_back(); allocator_->free(allocation.buffer, allocation.size); EXPECT_EQ(allocator_->getBytes(), currentBytes - allocation.size); - EXPECT_EQ(listener_->currentBytes(), 0); + EXPECT_EQ(listener_->currentBytes(), initBytes); ASSERT_EQ(allocator_->getBytes(), 0); } diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 02c6bf7fe4ac..1f682557bb18 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -1207,6 +1207,13 @@ object GlutenConfig { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("1GB") + val COLUMNAR_VELOX_MEM_INIT_CAPACITY = + buildConf("spark.gluten.sql.columnar.backend.velox.memInitCapacity") + .internal() + .doc("The initial memory capacity to reserve for a newly created Velox query memory pool.") + .bytesConf(ByteUnit.BYTE) + .createWithDefaultString("8MB") + val COLUMNAR_VELOX_SSD_CACHE_PATH = buildStaticConf("spark.gluten.sql.columnar.backend.velox.ssdCachePath") .internal() From c987bd20a3e48fb62370aedee2798f6700bf9775 Mon Sep 17 00:00:00 2001 From: James Xu Date: Wed, 22 May 2024 09:59:45 +0800 Subject: [PATCH 123/402] [GLUTEN-4039][VL] Implement stack function (#5813) We insert a ProjectExec before GenerateExec to organize stack's params as several arrays, these arrays then would be unnested using Unnest operator, for query: select stack(2, id, name, id1, name1) The plan is: Generate stack(2, id#122, name#123, id1#124, name1#125), false, [col0#137, col1#138] +- Project [id#122, name#123, id1#124, name1#125, array(id#122, id1#124) AS _pre_0#141, array(name#123,name1#125) AS _pre_1#142] +- RewrittenNodeWall LocalTableScan [id#122, name#123, id1#124, name1#125] --- .../gluten/utils/CHExpressionUtil.scala | 3 +- .../execution/GenerateExecTransformer.scala | 46 ++++++++++++- .../gluten/execution/TestOperator.scala | 37 +++++++++++ cpp/velox/substrait/SubstraitToVeloxPlan.cc | 64 ++++++++++++++----- docs/velox-backend-support-progress.md | 4 +- .../expression/ExpressionMappings.scala | 4 +- .../gluten/expression/ExpressionNames.scala | 1 + 7 files changed, 138 insertions(+), 21 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index 8454b1469009..5f78d25cc5c2 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -200,6 +200,7 @@ object CHExpressionUtil { TIMESTAMP_MILLIS -> DefaultValidator(), TIMESTAMP_MICROS -> DefaultValidator(), FLATTEN -> DefaultValidator(), - RINT -> DefaultValidator() + RINT -> DefaultValidator(), + STACK -> DefaultValidator() ) } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala index 8f57827423eb..23addb89ea89 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala @@ -109,6 +109,18 @@ case class GenerateExecTransformer( .append("isPosExplode=") .append(isPosExplode) .append("\n") + + // isStack: 1 for Stack, 0 for others. + val isStack = if (generator.isInstanceOf[Stack]) { + "1" + } else { + "0" + } + parametersStr + .append("isStack=") + .append(isStack) + .append("\n") + val message = StringValue .newBuilder() .setValue(parametersStr.toString) @@ -128,7 +140,7 @@ object GenerateExecTransformer { false } else { generator match { - case _: Inline | _: ExplodeBase | _: JsonTuple => + case _: Inline | _: ExplodeBase | _: JsonTuple | _: Stack => true case _ => false @@ -159,7 +171,7 @@ object PullOutGenerateProjectHelper extends PullOutProjectHelper { } val newGeneratorChildren = Seq(newGeneratorChild) - // Avoid using elimainateProjectList to create the project list + // Avoid using eliminateProjectList to create the project list // because newGeneratorChild can be a duplicated Attribute in generate.child.output. // The native side identifies the last field of projection as generator's input. generate.copy( @@ -167,6 +179,36 @@ object PullOutGenerateProjectHelper extends PullOutProjectHelper { generate.generator.withNewChildren(newGeneratorChildren).asInstanceOf[Generator], child = ProjectExec(generate.child.output ++ newGeneratorChildren, generate.child) ) + case stack: Stack => + val numRows = stack.children.head.eval().asInstanceOf[Int] + val numFields = Math.ceil((stack.children.size - 1.0) / numRows).toInt + + val newProjections = mutable.Buffer[NamedExpression]() + val args = stack.children.tail + + // We organize stack's params as `numFields` arrays which will be feed + // to Unnest operator on native side. + for (field <- 0 until numFields) { + val fieldArray = mutable.Buffer[Expression]() + + for (row <- 0 until numRows) { + val index = row * numFields + field + if (index < args.size) { + fieldArray += args(index) + } else { + // Append nulls. + fieldArray += Literal(null, args(field).dataType) + } + } + + newProjections += Alias(CreateArray(fieldArray), generatePreAliasName)() + } + + // Plug in a Project between Generate and its child. + generate.copy( + generator = generate.generator, + child = ProjectExec(generate.child.output ++ newProjections, generate.child) + ) case JsonTuple(Seq(jsonObj, jsonPaths @ _*)) => val getJsons: IndexedSeq[Expression] = { jsonPaths.map { diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 0872ac798382..287bf1e9bda2 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -803,6 +803,43 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + test("test stack function") { + withTempView("t1") { + sql("""SELECT * from values + | (1, "james", 10, "lucy"), + | (2, "bond", 20, "lily") + |as tbl(id, name, id1, name1) + """.stripMargin).createOrReplaceTempView("t1") + + // Stack function with attributes as params. + // Stack 4 attributes, no nulls need to be padded. + runQueryAndCompare(s""" + |SELECT stack(2, id, name, id1, name1) from t1; + |""".stripMargin) { + checkGlutenOperatorMatch[GenerateExecTransformer] + } + + // Stack 3 attributes: there will be nulls. + runQueryAndCompare(s""" + |SELECT stack(2, id, name, id1) from t1; + |""".stripMargin) { + checkGlutenOperatorMatch[GenerateExecTransformer] + } + + // Stack function with literals as params. + runQueryAndCompare("SELECT stack(2, 1, 2, 3);") { + checkGlutenOperatorMatch[GenerateExecTransformer] + } + + // Stack function with params mixed with attributes and literals. + runQueryAndCompare(s""" + |SELECT stack(2, id, name, 1) from t1; + |""".stripMargin) { + checkGlutenOperatorMatch[GenerateExecTransformer] + } + } + } + test("test inline function") { // Literal: func(literal) runQueryAndCompare(s""" diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 34ba6057c15f..c07826f5a8f3 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -706,6 +706,23 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: return std::make_shared(nextPlanNodeId(), projectSetExprs, std::move(names), childNode); } +namespace { + +void extractUnnestFieldExpr( + std::shared_ptr projNode, + int32_t index, + std::vector& unnestFields) { + auto name = projNode->names()[index]; + auto expr = projNode->projections()[index]; + auto type = expr->type(); + + auto unnestFieldExpr = std::make_shared(type, name); + VELOX_CHECK_NOT_NULL(unnestFieldExpr, " the key in unnest Operator only support field"); + unnestFields.emplace_back(unnestFieldExpr); +} + +} // namespace + core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait::GenerateRel& generateRel) { core::PlanNodePtr childNode; if (generateRel.has_input()) { @@ -732,22 +749,39 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: auto projNode = std::dynamic_pointer_cast(childNode); + bool isStack = generateRel.has_advanced_extension() && + SubstraitParser::configSetInOptimization(generateRel.advanced_extension(), "isStack="); + if (projNode != nullptr && projNode->names().size() > requiredChildOutput.size()) { - // Generator function's input is not a field reference, e.g. explode(array(1,2,3)), a sample - // input substrait plan is like the following(the plan structure is ensured by scala code): - // - // Generate explode([1,2,3] AS _pre_0#129), false, [col#126] - // +- Project [fake_column#128, [1,2,3] AS _pre_0#129] - // +- RewrittenNodeWall Scan OneRowRelation[fake_column#128] - // - // The last projection column in GeneratorRel's child(Project) is the column we need to unnest - auto innerName = projNode->names().back(); - auto innerExpr = projNode->projections().back(); - - auto innerType = innerExpr->type(); - auto unnestFieldExpr = std::make_shared(innerType, innerName); - VELOX_CHECK_NOT_NULL(unnestFieldExpr, " the key in unnest Operator only support field"); - unnest.emplace_back(unnestFieldExpr); + // Generator function's input is NOT a field reference. + if (!isStack) { + // For generator function which is not stack, e.g. explode(array(1,2,3)), a sample + // input substrait plan is like the following: + // + // Generate explode([1,2,3] AS _pre_0#129), false, [col#126] + // +- Project [fake_column#128, [1,2,3] AS _pre_0#129] + // +- RewrittenNodeWall Scan OneRowRelation[fake_column#128] + // The last projection column in GeneratorRel's child(Project) is the column we need to unnest + extractUnnestFieldExpr(projNode, projNode->projections().size() - 1, unnest); + } else { + // For stack function, e.g. stack(2, 1,2,3), a sample + // input substrait plan is like the following: + // + // Generate stack(2, id#122, name#123, id1#124, name1#125), false, [col0#137, col1#138] + // +- Project [id#122, name#123, id1#124, name1#125, array(id#122, id1#124) AS _pre_0#141, array(name#123, + // name1#125) AS _pre_1#142] + // +- RewrittenNodeWall LocalTableScan [id#122, name#123, id1#124, name1#125] + // + // The last `numFields` projections are the fields we want to unnest. + auto generatorFunc = generator.scalar_function(); + auto numRows = SubstraitParser::getLiteralValue(generatorFunc.arguments(0).value().literal()); + auto numFields = static_cast(std::ceil((generatorFunc.arguments_size() - 1.0) / numRows)); + auto totalProjectCount = projNode->names().size(); + + for (auto i = totalProjectCount - numFields; i < totalProjectCount; ++i) { + extractUnnestFieldExpr(projNode, i, unnest); + } + } } else { // Generator function's input is a field reference, e.g. explode(col), generator // function's first argument is the field reference we need to unnest. diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index b8acce266e9a..fb68740c7a22 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -273,7 +273,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | array_sort | array_sort | array_sort | S | | | | | | | | | | | | | | | | | | | | | array_union | | | | | | | | | | | | | | | | | | | | | | | | arrays_overlap | array_overlap | | | | | | | | | | | | | | | | | | | | | | -| arrays_zip | zip | | S | | | | | | | | | | | | | | | | | | | | +| arrays_zip | zip | | S | | | | | | | | | | | | | | | | | | | | | cardinality | cardinality | | | | | | | | | | | | | | | | | | | | | | | element_at | element_at | element_at | S | | | | | | | | | | | | | | | | S | S | | | | exists | any_match | | S | | | | | | | | | | | | | | | | | | | | @@ -446,6 +446,6 @@ Gluten supports 199 functions. (Drag to right to see all data types) | sha1 | sha1 | sha1 | S | | | | | | | | | | | S | | | | | | | | | | sha2 | | sha2 | S | | | | | | | | | | | S | | | | | | | | | | spark_partition_id | | | S | | | | | | | | | | | | | | | | | | | | -| stack | | | | | | | | | | | | | | | | | | | | | | | +| stack | | | S | | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | | xxhash64 | xxhash64 | xxhash64 | | | | | | | | | | | | | | | | | | | | | | uuid | uuid | uuid | S | | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index f910c8a98e29..c734967dea68 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -276,7 +276,9 @@ object ExpressionMappings { Sig[MonotonicallyIncreasingID](MONOTONICALLY_INCREASING_ID), Sig[SparkPartitionID](SPARK_PARTITION_ID), // Decimal - Sig[UnscaledValue](UNSCALED_VALUE) + Sig[UnscaledValue](UNSCALED_VALUE), + // Generator function + Sig[Stack](STACK) ) ++ SparkShimLoader.getSparkShims.scalarExpressionMappings /** Mapping Spark aggregate expression to Substrait function name */ diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 1b73b2686952..eded85e06006 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -294,6 +294,7 @@ object ExpressionNames { final val AGGREGATE = "aggregate" final val LAMBDAFUNCTION = "lambdafunction" final val EXPLODE = "explode" + final val STACK = "stack" final val INLINE = "inline" final val POSEXPLODE = "posexplode" final val CHECK_OVERFLOW = "check_overflow" From a533a062dd788c4be934d582e32529b41e1ffe78 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 22 May 2024 10:21:51 +0800 Subject: [PATCH 124/402] [VL] Remove unused code for sort based shuffle (#5826) --- cpp/core/CMakeLists.txt | 1 - cpp/core/jni/JniWrapper.cc | 13 -------- cpp/core/shuffle/Options.h | 2 +- cpp/core/shuffle/rss/RemotePartitionWriter.cc | 20 ------------ cpp/core/shuffle/rss/RemotePartitionWriter.h | 32 ------------------- cpp/core/shuffle/rss/RssPartitionWriter.h | 6 ++-- .../shuffle/VeloxSortBasedShuffleWriter.cc | 6 +--- .../shuffle/VeloxSortBasedShuffleWriter.h | 12 ++----- 8 files changed, 7 insertions(+), 85 deletions(-) delete mode 100644 cpp/core/shuffle/rss/RemotePartitionWriter.cc delete mode 100644 cpp/core/shuffle/rss/RemotePartitionWriter.h diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index e2d312abaacb..dc9ce3435c38 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -191,7 +191,6 @@ set(SPARK_COLUMNAR_PLUGIN_SRCS shuffle/Partitioner.cc shuffle/Partitioning.cc shuffle/Payload.cc - shuffle/rss/RemotePartitionWriter.cc shuffle/rss/RssPartitionWriter.cc shuffle/RoundRobinPartitioner.cc shuffle/ShuffleMemoryPool.cc diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 6a1926317071..b1edfbd019b2 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -850,19 +850,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe .startPartitionId = startPartitionId, }; - jclass cls = env->FindClass("java/lang/Thread"); - jmethodID mid = env->GetStaticMethodID(cls, "currentThread", "()Ljava/lang/Thread;"); - jobject thread = env->CallStaticObjectMethod(cls, mid); - checkException(env); - if (thread == NULL) { - LOG(WARNING) << "Thread.currentThread() return NULL"; - } else { - jmethodID midGetid = getMethodIdOrError(env, cls, "getId", "()J"); - jlong sid = env->CallLongMethod(thread, midGetid); - checkException(env); - shuffleWriterOptions.threadId = (int64_t)sid; - } - auto partitionWriterOptions = PartitionWriterOptions{ .mergeBufferSize = mergeBufferSize, .mergeThreshold = mergeThreshold, diff --git a/cpp/core/shuffle/Options.h b/cpp/core/shuffle/Options.h index 4317ed6318ed..4828c7c8229d 100644 --- a/cpp/core/shuffle/Options.h +++ b/cpp/core/shuffle/Options.h @@ -26,7 +26,7 @@ namespace gluten { static constexpr int16_t kDefaultBatchSize = 4096; static constexpr int32_t kDefaultShuffleWriterBufferSize = 4096; -static constexpr int64_t kDefaultSortBufferThreshold = 64000000000; +static constexpr int64_t kDefaultSortBufferThreshold = 64 << 20; static constexpr int64_t kDefaultPushMemoryThreshold = 4096; static constexpr int32_t kDefaultNumSubDirs = 64; static constexpr int32_t kDefaultCompressionThreshold = 100; diff --git a/cpp/core/shuffle/rss/RemotePartitionWriter.cc b/cpp/core/shuffle/rss/RemotePartitionWriter.cc deleted file mode 100644 index 9993956b6472..000000000000 --- a/cpp/core/shuffle/rss/RemotePartitionWriter.cc +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "RemotePartitionWriter.h" - -namespace gluten {} // namespace gluten diff --git a/cpp/core/shuffle/rss/RemotePartitionWriter.h b/cpp/core/shuffle/rss/RemotePartitionWriter.h deleted file mode 100644 index 477166635d72..000000000000 --- a/cpp/core/shuffle/rss/RemotePartitionWriter.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "shuffle/PartitionWriter.h" - -#include - -namespace gluten { - -class RemotePartitionWriter : public PartitionWriter { - public: - explicit RemotePartitionWriter(uint32_t numPartitions, PartitionWriterOptions options, arrow::MemoryPool* pool) - : PartitionWriter(numPartitions, std::move(options), pool) {} -}; - -} // namespace gluten diff --git a/cpp/core/shuffle/rss/RssPartitionWriter.h b/cpp/core/shuffle/rss/RssPartitionWriter.h index b8cc1551c0cd..d993aa9eab8e 100644 --- a/cpp/core/shuffle/rss/RssPartitionWriter.h +++ b/cpp/core/shuffle/rss/RssPartitionWriter.h @@ -20,20 +20,20 @@ #include #include -#include "shuffle/rss/RemotePartitionWriter.h" +#include "shuffle/PartitionWriter.h" #include "shuffle/rss/RssClient.h" #include "utils/macros.h" namespace gluten { -class RssPartitionWriter final : public RemotePartitionWriter { +class RssPartitionWriter final : public PartitionWriter { public: RssPartitionWriter( uint32_t numPartitions, PartitionWriterOptions options, arrow::MemoryPool* pool, std::shared_ptr rssClient) - : RemotePartitionWriter(numPartitions, std::move(options), pool), rssClient_(rssClient) { + : PartitionWriter(numPartitions, std::move(options), pool), rssClient_(rssClient) { init(); } diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc index 2a6bca8c0f37..85fccf7165bd 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc @@ -52,7 +52,6 @@ arrow::Status VeloxSortBasedShuffleWriter::init() { partitioner_, Partitioner::make(options_.partitioning, numPartitions_, options_.startPartitionId)); DLOG(INFO) << "Create partitioning type: " << std::to_string(options_.partitioning); - partition2RowCount_.resize(numPartitions_); rowVectorIndexMap_.reserve(numPartitions_); for (auto pid = 0; pid < numPartitions_; ++pid) { rowVectorIndexMap_[pid].reserve(options_.bufferSize); @@ -68,7 +67,6 @@ arrow::Status VeloxSortBasedShuffleWriter::doSort(facebook::velox::RowVectorPtr if (currentInputColumnBytes_ > memLimit) { for (auto pid = 0; pid < numPartitions(); ++pid) { RETURN_NOT_OK(evictRowVector(pid)); - partition2RowCount_[pid] = 0; } batches_.clear(); currentInputColumnBytes_ = 0; @@ -77,7 +75,7 @@ arrow::Status VeloxSortBasedShuffleWriter::doSort(facebook::velox::RowVectorPtr return arrow::Status::OK(); } -arrow::Status VeloxSortBasedShuffleWriter::write(std::shared_ptr cb, int64_t memLimit) { +arrow::Status VeloxSortBasedShuffleWriter::write(std::shared_ptr cb, int64_t /* memLimit */) { if (options_.partitioning == Partitioning::kSingle) { auto veloxColumnBatch = VeloxColumnarBatch::from(veloxPool_.get(), cb); VELOX_CHECK_NOT_NULL(veloxColumnBatch); @@ -199,7 +197,6 @@ arrow::Status VeloxSortBasedShuffleWriter::evictRowVector(uint32_t partitionId) arrow::Status VeloxSortBasedShuffleWriter::stop() { for (auto pid = 0; pid < numPartitions(); ++pid) { RETURN_NOT_OK(evictRowVector(pid)); - partition2RowCount_[pid] = 0; } batches_.clear(); currentInputColumnBytes_ = 0; @@ -236,7 +233,6 @@ arrow::Status VeloxSortBasedShuffleWriter::reclaimFixedSize(int64_t size, int64_ if (sortState_ == SortState::kSortInit) { for (auto pid = 0; pid < numPartitions(); ++pid) { RETURN_NOT_OK(evictRowVector(pid)); - partition2RowCount_[pid] = 0; } batches_.clear(); *actual = currentInputColumnBytes_; diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h index 417d5e926012..bb50021fe548 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h @@ -49,7 +49,7 @@ namespace gluten { enum SortState { kSortInit, kSort, kSortStop }; -class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { +class VeloxSortBasedShuffleWriter final : public VeloxShuffleWriter { public: static arrow::Result> create( uint32_t numPartitions, @@ -81,7 +81,7 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { void setSortState(SortState state); - arrow::Status doSort(facebook::velox::RowVectorPtr rv, int64_t memLimit); + arrow::Status doSort(facebook::velox::RowVectorPtr rv, int64_t /* memLimit */); arrow::Status evictBatch(uint32_t partitionId); @@ -92,12 +92,6 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { std::unique_ptr batch_; std::unique_ptr bufferOutputStream_; - // Partition ID -> Row Count - // subscript: Partition ID - // value: How many rows does this partition have in the current input RowVector - // Updated for each input RowVector. - std::vector partition2RowCount_; - std::unique_ptr serde_ = std::make_unique(); @@ -105,8 +99,6 @@ class VeloxSortBasedShuffleWriter : public VeloxShuffleWriter { std::unordered_map> rowVectorIndexMap_; - std::unordered_map> rowVectorPartitionMap_; - uint32_t currentInputColumnBytes_ = 0; SortState sortState_{kSortInit}; From 2dd62f6f9d0ede63f47bcfe92f731c0169d2131f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Wed, 22 May 2024 14:09:46 +0800 Subject: [PATCH 125/402] [VL] Not fallback for function spark_partition_id (#5830) [VL] Not fallback for function spark_partition_id. --- .../org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 4 ++-- .../gluten/execution/ScalarFunctionsValidateSuite.scala | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index c16b3624f319..a2da0b8b2a86 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -27,7 +27,7 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat.{DwrfReadFormat, OrcReadFormat, ParquetReadFormat} import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, Expression, Lag, Lead, Literal, MakeYMInterval, NamedExpression, NthValue, NTile, PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid} +import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, Expression, Lag, Lead, Literal, MakeYMInterval, NamedExpression, NthValue, NTile, PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder, SparkPartitionID, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, ApproximatePercentile, Count, Sum} import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.util.CharVarcharUtils @@ -416,7 +416,7 @@ object VeloxBackendSettings extends BackendSettingsApi { expr match { // Block directly falling back the below functions by FallbackEmptySchemaRelation. case alias: Alias => checkExpr(alias.child) - case _: Rand | _: Uuid | _: MakeYMInterval => true + case _: Rand | _: Uuid | _: MakeYMInterval | _: SparkPartitionID => true case _ => false } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index d0df35b64e25..b3753ab8352c 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -588,6 +588,10 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { | from lineitem limit 100""".stripMargin) { checkGlutenOperatorMatch[ProjectExecTransformer] } + runQueryAndCompare("""SELECT spark_partition_id() + |from lineitem limit 100""".stripMargin) { + checkGlutenOperatorMatch[ProjectExecTransformer] + } } testWithSpecifiedSparkVersion("Test url_decode function", Some("3.4")) { From d966ea7f4ad86aa4a974f3fa71269e8c2ba385e6 Mon Sep 17 00:00:00 2001 From: James Xu Date: Wed, 22 May 2024 15:20:48 +0800 Subject: [PATCH 126/402] [GLUTEN-5837][VL] Fix duplicated projection name during substrait GenerateRel conversion (#5838) --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index c07826f5a8f3..2362030661d0 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -794,15 +794,14 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: unnest.emplace_back(unnestFieldExpr); } - // TODO(yuan): get from generator output std::vector unnestNames; int unnestIndex = 0; for (const auto& variable : unnest) { if (variable->type()->isArray()) { - unnestNames.emplace_back(fmt::format("C{}", unnestIndex++)); + unnestNames.emplace_back(SubstraitParser::makeNodeName(planNodeId_, unnestIndex++)); } else if (variable->type()->isMap()) { - unnestNames.emplace_back(fmt::format("C{}", unnestIndex++)); - unnestNames.emplace_back(fmt::format("C{}", unnestIndex++)); + unnestNames.emplace_back(SubstraitParser::makeNodeName(planNodeId_, unnestIndex++)); + unnestNames.emplace_back(SubstraitParser::makeNodeName(planNodeId_, unnestIndex++)); } else { VELOX_FAIL( "Unexpected type of unnest variable. Expected ARRAY or MAP, but got {}.", variable->type()->toString()); From 3e33591c145048d4cda10de9deb1f8c7b370552c Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Wed, 22 May 2024 02:59:45 -0500 Subject: [PATCH 127/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240522) (#5835) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240522) * Fxi build due to https://github.com/ClickHouse/ClickHouse/pull/64108 which move StringUtils.h from src/Common/StringUtils/StringUtils.h to src/Common/StringUtils.h. To avoid conflict, we rename StringUtils.h to GlutenStringUtils.h (cherry picked from commit 60f2b9e16accafd97e3135feb1036d877dbb363e) --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- .../Common/{StringUtils.cpp => GlutenStringUtils.cpp} | 6 +++--- .../Common/{StringUtils.h => GlutenStringUtils.h} | 2 +- .../Operator/PartitionColumnFillingTransform.cpp | 4 ++-- cpp-ch/local-engine/Parser/AggregateRelParser.cpp | 4 ---- .../Storages/Serializations/ExcelReadHelpers.cpp | 4 +--- .../local-engine/Storages/Serializations/ExcelReadHelpers.h | 4 ++-- .../Storages/Serializations/ExcelStringReader.cpp | 2 +- cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp | 6 +++--- .../Storages/SubstraitSource/SubstraitFileSource.cpp | 4 ++-- cpp-ch/local-engine/tests/gtest_utils.cpp | 4 ++-- 11 files changed, 19 insertions(+), 25 deletions(-) rename cpp-ch/local-engine/Common/{StringUtils.cpp => GlutenStringUtils.cpp} (87%) rename cpp-ch/local-engine/Common/{StringUtils.h => GlutenStringUtils.h} (97%) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 6a58dce0f4c8..04bd6960ee83 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240515 -CH_COMMIT=00867009134 \ No newline at end of file +CH_BRANCH=rebase_ch/20240522 +CH_COMMIT=327f885e4bd \ No newline at end of file diff --git a/cpp-ch/local-engine/Common/StringUtils.cpp b/cpp-ch/local-engine/Common/GlutenStringUtils.cpp similarity index 87% rename from cpp-ch/local-engine/Common/StringUtils.cpp rename to cpp-ch/local-engine/Common/GlutenStringUtils.cpp index 6ac6e57afca1..b6d11ac1b267 100644 --- a/cpp-ch/local-engine/Common/StringUtils.cpp +++ b/cpp-ch/local-engine/Common/GlutenStringUtils.cpp @@ -14,14 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "StringUtils.h" +#include "GlutenStringUtils.h" #include #include #include namespace local_engine { -PartitionValues StringUtils::parsePartitionTablePath(const std::string & file) +PartitionValues GlutenStringUtils::parsePartitionTablePath(const std::string & file) { PartitionValues result; Poco::StringTokenizer path(file, "/"); @@ -35,7 +35,7 @@ PartitionValues StringUtils::parsePartitionTablePath(const std::string & file) } return result; } -bool StringUtils::isNullPartitionValue(const std::string & value) +bool GlutenStringUtils::isNullPartitionValue(const std::string & value) { return value == "__HIVE_DEFAULT_PARTITION__"; } diff --git a/cpp-ch/local-engine/Common/StringUtils.h b/cpp-ch/local-engine/Common/GlutenStringUtils.h similarity index 97% rename from cpp-ch/local-engine/Common/StringUtils.h rename to cpp-ch/local-engine/Common/GlutenStringUtils.h index 64e0ee6db7ac..023cb2b8d047 100644 --- a/cpp-ch/local-engine/Common/StringUtils.h +++ b/cpp-ch/local-engine/Common/GlutenStringUtils.h @@ -23,7 +23,7 @@ namespace local_engine using PartitionValue = std::pair; using PartitionValues = std::vector; -class StringUtils +class GlutenStringUtils { public: static PartitionValues parsePartitionTablePath(const std::string & file); diff --git a/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.cpp b/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.cpp index f73b06dedc5b..724a02f5b296 100644 --- a/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.cpp +++ b/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include using namespace DB; @@ -77,7 +77,7 @@ ColumnPtr PartitionColumnFillingTransform::createPartitionColumn() if (const DataTypeNullable * nullable_type = checkAndGetDataType(partition_col_type.get())) { nested_type = nullable_type->getNestedType(); - if (StringUtils::isNullPartitionValue(partition_col_value)) + if (GlutenStringUtils::isNullPartitionValue(partition_col_value)) { return nullable_type->createColumnConstWithDefaultValue(1); } diff --git a/cpp-ch/local-engine/Parser/AggregateRelParser.cpp b/cpp-ch/local-engine/Parser/AggregateRelParser.cpp index d20f30e41191..0857995571d4 100644 --- a/cpp-ch/local-engine/Parser/AggregateRelParser.cpp +++ b/cpp-ch/local-engine/Parser/AggregateRelParser.cpp @@ -18,9 +18,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -31,8 +29,6 @@ #include #include #include -#include "Common/PODArray.h" -#include namespace DB { diff --git a/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.cpp b/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.cpp index 621349732ee9..6a7b7b2e29cc 100644 --- a/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.cpp +++ b/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.cpp @@ -16,11 +16,9 @@ */ #include "ExcelReadHelpers.h" - #include #include -#include - +#include namespace DB { diff --git a/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.h b/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.h index cf5aeda6f84f..f96b31f7049d 100644 --- a/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.h +++ b/cpp-ch/local-engine/Storages/Serializations/ExcelReadHelpers.h @@ -19,10 +19,10 @@ #include #include #include -#include + #include -#include +#include #include "ExcelNumberReader.h" diff --git a/cpp-ch/local-engine/Storages/Serializations/ExcelStringReader.cpp b/cpp-ch/local-engine/Storages/Serializations/ExcelStringReader.cpp index 9aac8d0af2d5..72637f91048e 100644 --- a/cpp-ch/local-engine/Storages/Serializations/ExcelStringReader.cpp +++ b/cpp-ch/local-engine/Storages/Serializations/ExcelStringReader.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include "ExcelStringReader.h" diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp index 8199107998d2..0221afd88514 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp @@ -18,9 +18,9 @@ #include #include -#include "Common/CHUtil.h" +#include #include -#include +#include #include #if USE_PARQUET @@ -53,7 +53,7 @@ FormatFile::FormatFile( const ReadBufferBuilderPtr & read_buffer_builder_) : context(context_), file_info(file_info_), read_buffer_builder(read_buffer_builder_) { - PartitionValues part_vals = StringUtils::parsePartitionTablePath(file_info.uri_file()); + PartitionValues part_vals = GlutenStringUtils::parsePartitionTablePath(file_info.uri_file()); String partition_values_str = "["; for (size_t i = 0; i < part_vals.size(); ++i) { diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp index 44da624d408a..80dccf759060 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include "DataTypes/DataTypesDecimal.h" @@ -152,7 +152,7 @@ DB::ColumnPtr FileReaderWrapper::createConstColumn(DB::DataTypePtr data_type, co DB::ColumnPtr FileReaderWrapper::createColumn(const String & value, DB::DataTypePtr type, size_t rows) { - if (StringUtils::isNullPartitionValue(value)) + if (GlutenStringUtils::isNullPartitionValue(value)) { if (!type->isNullable()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Partition column is null value,but column data type is not nullable."); diff --git a/cpp-ch/local-engine/tests/gtest_utils.cpp b/cpp-ch/local-engine/tests/gtest_utils.cpp index b01656eb7567..4ea713921f6a 100644 --- a/cpp-ch/local-engine/tests/gtest_utils.cpp +++ b/cpp-ch/local-engine/tests/gtest_utils.cpp @@ -15,14 +15,14 @@ * limitations under the License. */ #include -#include +#include using namespace local_engine; TEST(TestStringUtils, TestExtractPartitionValues) { std::string path = "/tmp/col1=1/col2=test/a.parquet"; - auto values = StringUtils::parsePartitionTablePath(path); + auto values = GlutenStringUtils::parsePartitionTablePath(path); ASSERT_EQ(2, values.size()); ASSERT_EQ("col1", values[0].first); ASSERT_EQ("1", values[0].second); From d6b298221f1360626e52862985f78abc7436183d Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 22 May 2024 17:55:00 +0800 Subject: [PATCH 128/402] [VL] RAS: Reuse same code path with heuristic planner for convention enforcement (#5824) --- .../velox/VeloxSparkPlanExecApi.scala | 3 +- .../apache/gluten/planner/VeloxRasSuite.scala | 15 +- .../execution/ColumnarToRowExecBase.scala | 10 +- .../apache/gluten/extension/GlutenPlan.scala | 3 +- .../enumerated/EnumeratedTransform.scala | 13 +- .../columnar/transition/Convention.scala | 4 + .../columnar/transition/ConventionFunc.scala | 115 +++++++++++--- .../columnar/transition/ConventionReq.scala | 17 +- .../columnar/transition/Transition.scala | 30 +++- .../columnar/transition/Transitions.scala | 48 +----- .../gluten/planner/plan/GlutenPlanModel.scala | 70 +++++++-- .../apache/gluten/planner/property/Conv.scala | 106 +++++++++++++ .../gluten/planner/property/Convention.scala | 147 ------------------ .../property/GlutenPropertyModel.scala | 6 +- .../org/apache/spark/util/SparkTaskUtil.scala | 29 ++++ .../gluten/columnarbatch/ArrowBatch.scala | 7 +- .../org/apache/gluten/ras/PlanModel.scala | 2 +- .../scala/org/apache/gluten/ras/Ras.scala | 22 +-- .../org/apache/gluten/ras/RasGroup.scala | 10 +- .../scala/org/apache/gluten/ras/RasNode.scala | 6 +- .../ras/exaustive/ExhaustivePlanner.scala | 2 +- .../apache/gluten/ras/rule/RuleApplier.scala | 2 +- .../gluten/ras/vis/GraphvizVisualizer.scala | 2 +- .../apache/gluten/ras/OperationSuite.scala | 7 +- .../org/apache/gluten/ras/PropertySuite.scala | 9 +- .../org/apache/gluten/ras/RasSuiteBase.scala | 8 +- .../gluten/ras/mock/MockMemoState.scala | 2 +- .../apache/gluten/ras/mock/MockRasPath.scala | 2 +- .../ras/specific/DistributedSuite.scala | 18 ++- 29 files changed, 421 insertions(+), 294 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala create mode 100644 gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 2d37b118592d..3221165827ca 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -81,7 +81,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { */ override def batchTypeFunc(): BatchOverride = { case i: InMemoryTableScanExec - if i.relation.cacheBuilder.serializer.isInstanceOf[ColumnarCachedBatchSerializer] => + if i.supportsColumnar && i.relation.cacheBuilder.serializer + .isInstanceOf[ColumnarCachedBatchSerializer] => VeloxBatch } diff --git a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala index 4690ef5168a3..ae2cea0ba03d 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala @@ -16,7 +16,8 @@ */ package org.apache.gluten.planner -import org.apache.gluten.planner.property.Conventions +import org.apache.gluten.extension.columnar.transition.ConventionReq +import org.apache.gluten.planner.property.Conv import org.apache.gluten.ras.Best.BestNotFoundException import org.apache.gluten.ras.Ras import org.apache.gluten.ras.RasSuiteBase._ @@ -44,7 +45,7 @@ class VeloxRasSuite extends SharedSparkSession { test("C2R, R2C - explicitly requires any properties") { val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) val planner = - newRas().newPlanner(in, PropertySet(List(Conventions.ANY))) + newRas().newPlanner(in, PropertySet(List(Conv.any))) val out = planner.plan() assert(out == RowUnary(RowLeaf(TRIVIAL_SCHEMA))) } @@ -52,7 +53,7 @@ class VeloxRasSuite extends SharedSparkSession { test("C2R, R2C - requires columnar output") { val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) val planner = - newRas().newPlanner(in, PropertySet(List(Conventions.VANILLA_COLUMNAR))) + newRas().newPlanner(in, PropertySet(List(Conv.req(ConventionReq.vanillaBatch)))) val out = planner.plan() assert(out == RowToColumnarExec(RowUnary(RowLeaf(TRIVIAL_SCHEMA)))) } @@ -63,7 +64,7 @@ class VeloxRasSuite extends SharedSparkSession { RowUnary( RowUnary(ColumnarUnary(RowUnary(RowUnary(ColumnarUnary(RowLeaf(TRIVIAL_SCHEMA)))))))) val planner = - newRas().newPlanner(in, PropertySet(List(Conventions.ROW_BASED))) + newRas().newPlanner(in, PropertySet(List(Conv.req(ConventionReq.row)))) val out = planner.plan() assert( out == ColumnarToRowExec( @@ -91,7 +92,7 @@ class VeloxRasSuite extends SharedSparkSession { RowUnary(ColumnarUnary(RowUnary(RowUnary(ColumnarUnary(RowLeaf(TRIVIAL_SCHEMA)))))))) val planner = newRas(List(ConvertRowUnaryToColumnar)) - .newPlanner(in, PropertySet(List(Conventions.ROW_BASED))) + .newPlanner(in, PropertySet(List(Conv.req(ConventionReq.row)))) val out = planner.plan() assert(out == ColumnarToRowExec(ColumnarUnary(ColumnarUnary(ColumnarUnary(ColumnarUnary( ColumnarUnary(ColumnarUnary(ColumnarUnary(RowToColumnarExec(RowLeaf(TRIVIAL_SCHEMA))))))))))) @@ -104,7 +105,7 @@ class VeloxRasSuite extends SharedSparkSession { val in = RowUnary(RowLeaf(EMPTY_SCHEMA)) val planner = - newRas().newPlanner(in, PropertySet(List(Conventions.ANY))) + newRas().newPlanner(in, PropertySet(List(Conv.any))) val out = planner.plan() assert(out == RowUnary(RowLeaf(EMPTY_SCHEMA))) @@ -112,7 +113,7 @@ class VeloxRasSuite extends SharedSparkSession { // Could not optimize to columnar output since R2C transitions for empty schema node // is not allowed. val planner2 = - newRas().newPlanner(in, PropertySet(List(Conventions.VANILLA_COLUMNAR))) + newRas().newPlanner(in, PropertySet(List(Conv.req(ConventionReq.vanillaBatch)))) planner2.plan() } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala index 6d3fa2dac609..fd86106bf367 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarToRowExecBase.scala @@ -18,6 +18,8 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.transition.ConventionReq +import org.apache.gluten.extension.columnar.transition.ConventionReq.KnownChildrenConventions import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD @@ -28,7 +30,8 @@ import org.apache.spark.sql.execution.{ColumnarToRowTransition, SparkPlan} abstract class ColumnarToRowExecBase(child: SparkPlan) extends ColumnarToRowTransition - with GlutenPlan { + with GlutenPlan + with KnownChildrenConventions { // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics = @@ -50,4 +53,9 @@ abstract class ColumnarToRowExecBase(child: SparkPlan) override def doExecute(): RDD[InternalRow] = { doExecuteInternal() } + + override def requiredChildrenConventions(): Seq[ConventionReq] = { + List(ConventionReq.backendBatch) + } + } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala index 033e44b8c4d2..8f1004be4aaa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala @@ -88,8 +88,7 @@ trait GlutenPlan extends SparkPlan with Convention.KnownBatchType with LogLevelU final override def batchType(): Convention.BatchType = { if (!supportsColumnar) { - throw new UnsupportedOperationException( - s"Node $nodeName doesn't support columnar-batch processing") + return Convention.BatchType.None } val batchType = batchType0() assert(batchType != Convention.BatchType.None) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index dc34bc1af2a4..50f0dce13dc7 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -17,8 +17,9 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.extension.columnar.{OffloadExchange, OffloadJoin, OffloadOthers, OffloadSingleNode} +import org.apache.gluten.extension.columnar.transition.ConventionReq import org.apache.gluten.planner.GlutenOptimization -import org.apache.gluten.planner.property.Conventions +import org.apache.gluten.planner.property.Conv import org.apache.gluten.ras.property.PropertySet import org.apache.gluten.utils.LogLevelUtil @@ -48,9 +49,13 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) private val optimization = GlutenOptimization(rules ++ offloadRules) - private val reqConvention = Conventions.ANY - private val altConventions = - Seq(Conventions.GLUTEN_COLUMNAR, Conventions.ROW_BASED) + private val reqConvention = Conv.any + + private val altConventions = { + val rowBased: Conv = Conv.req(ConventionReq.row) + val backendBatchBased: Conv = Conv.req(ConventionReq.backendBatch) + Seq(rowBased, backendBatchBased) + } override def apply(plan: SparkPlan): SparkPlan = { val constraintSet = PropertySet(List(reqConvention)) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala index 2774497d9c22..034b45851455 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Convention.scala @@ -110,4 +110,8 @@ object Convention { trait KnownBatchType { def batchType(): BatchType } + + trait KnownRowType { + def rowType(): RowType + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala index 28bd1d12caf3..453df5d88135 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionFunc.scala @@ -17,20 +17,22 @@ package org.apache.gluten.extension.columnar.transition import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.extension.columnar.transition.Convention.{BatchType, RowType} +import org.apache.gluten.extension.columnar.transition.ConventionReq.KnownChildrenConventions import org.apache.gluten.sql.shims.SparkShimLoader -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan, UnionExec} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} +import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.exchange.ReusedExchangeExec -/** ConventionFunc is a utility to derive [[Convention]] from a query plan. */ +/** ConventionFunc is a utility to derive [[Convention]] or [[ConventionReq]] from a query plan. */ trait ConventionFunc { def conventionOf(plan: SparkPlan): Convention + def conventionReqOf(plan: SparkPlan): ConventionReq } object ConventionFunc { - type BatchOverride = PartialFunction[SparkPlan, BatchType] + type BatchOverride = PartialFunction[SparkPlan, Convention.BatchType] // For testing, to make things work without a backend loaded. private var ignoreBackend: Boolean = false @@ -47,18 +49,22 @@ object ConventionFunc { } def create(): ConventionFunc = { + val batchOverride = newOverride() + new BuiltinFunc(batchOverride) + } + + private def newOverride(): BatchOverride = { synchronized { if (ignoreBackend) { // For testing - return new BuiltinFunc(PartialFunction.empty) + return PartialFunction.empty } } - val batchOverride = BackendsApiManager.getSparkPlanExecApiInstance.batchTypeFunc() - new BuiltinFunc(batchOverride) + BackendsApiManager.getSparkPlanExecApiInstance.batchTypeFunc() } private class BuiltinFunc(o: BatchOverride) extends ConventionFunc { - + import BuiltinFunc._ override def conventionOf(plan: SparkPlan): Convention = { val conv = conventionOf0(plan) conv @@ -82,7 +88,7 @@ object ConventionFunc { // See org.apache.gluten.extension.columnar.transition.InsertTransitions.apply BackendsApiManager.getSparkPlanExecApiInstance.batchType } else { - BatchType.None + Convention.BatchType.None } val conv = Convention.of(rowType, batchType) conv @@ -91,25 +97,94 @@ object ConventionFunc { conv } - private def rowTypeOf(plan: SparkPlan): RowType = { - if (!SparkShimLoader.getSparkShims.supportsRowBased(plan)) { - return RowType.None + private def rowTypeOf(plan: SparkPlan): Convention.RowType = { + val out = plan match { + case k: Convention.KnownRowType => + k.rowType() + case _ if !SparkShimLoader.getSparkShims.supportsRowBased(plan) => + Convention.RowType.None + case _ => + Convention.RowType.VanillaRow } - RowType.VanillaRow + if (out != Convention.RowType.None) { + assert(SparkShimLoader.getSparkShims.supportsRowBased(plan)) + } + out } - private def batchTypeOf(plan: SparkPlan): BatchType = { - if (!plan.supportsColumnar) { - return BatchType.None - } - o.applyOrElse( + private def batchTypeOf(plan: SparkPlan): Convention.BatchType = { + val out = o.applyOrElse( plan, (p: SparkPlan) => p match { - case g: Convention.KnownBatchType => g.batchType() - case _ => BatchType.VanillaBatch + case k: Convention.KnownBatchType => + k.batchType() + case _ if !plan.supportsColumnar => + Convention.BatchType.None + case _ => + Convention.BatchType.VanillaBatch } ) + if (out != Convention.BatchType.None) { + assert(plan.supportsColumnar) + } + out + } + + override def conventionReqOf(plan: SparkPlan): ConventionReq = { + val out = conventionReqOf0(plan) + out + } + + private def conventionReqOf0(plan: SparkPlan): ConventionReq = plan match { + case k: KnownChildrenConventions => + val reqs = k.requiredChildrenConventions().distinct + // This can be a temporary restriction. + assert( + reqs.size == 1, + "KnownChildrenConventions#requiredChildrenConventions should output the same element" + + " for all children") + reqs.head + case RowToColumnarLike(_) => + ConventionReq.of( + ConventionReq.RowType.Is(Convention.RowType.VanillaRow), + ConventionReq.BatchType.Any) + case ColumnarToRowExec(_) => + ConventionReq.of( + ConventionReq.RowType.Any, + ConventionReq.BatchType.Is(Convention.BatchType.VanillaBatch)) + case write: DataWritingCommandExec if SparkShimLoader.getSparkShims.isPlannedV1Write(write) => + // To align with ApplyColumnarRulesAndInsertTransitions#insertTransitions + ConventionReq.any + case u: UnionExec => + // We force vanilla union to output row data to get best compatibility with vanilla Spark. + // As a result it's a common practice to rewrite it with GlutenPlan for offloading. + ConventionReq.of( + ConventionReq.RowType.Is(Convention.RowType.VanillaRow), + ConventionReq.BatchType.Any) + case other => + // In the normal case, children's convention should follow parent node's convention. + // Note, we don't have consider C2R / R2C here since they are already removed by + // RemoveTransitions. + val thisConv = conventionOf0(other) + thisConv.asReq() + } + } + + private object BuiltinFunc { + implicit private class ConventionOps(conv: Convention) { + def asReq(): ConventionReq = { + val rowTypeReq = conv.rowType match { + case Convention.RowType.None => ConventionReq.RowType.Any + case r => ConventionReq.RowType.Is(r) + } + + val batchTypeReq = conv.batchType match { + case Convention.BatchType.None => ConventionReq.BatchType.Any + case b => ConventionReq.BatchType.Is(b) + } + ConventionReq.of(rowTypeReq, batchTypeReq) + } } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala index aac2084a7a7e..65422b38070e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/ConventionReq.scala @@ -16,6 +16,10 @@ */ package org.apache.gluten.extension.columnar.transition +import org.apache.gluten.backendsapi.BackendsApiManager + +import org.apache.spark.sql.execution.SparkPlan + /** * ConventionReq describes the requirement for [[Convention]]. This is mostly used in determining * the acceptable conventions for its children of a parent plan node. @@ -50,5 +54,16 @@ object ConventionReq { ) extends ConventionReq val any: ConventionReq = Impl(RowType.Any, BatchType.Any) - def of(rowType: RowType, batchType: BatchType): ConventionReq = new Impl(rowType, batchType) + val row: ConventionReq = Impl(RowType.Is(Convention.RowType.VanillaRow), BatchType.Any) + val vanillaBatch: ConventionReq = + Impl(RowType.Any, BatchType.Is(Convention.BatchType.VanillaBatch)) + lazy val backendBatch: ConventionReq = + Impl(RowType.Any, BatchType.Is(BackendsApiManager.getSparkPlanExecApiInstance.batchType)) + + def get(plan: SparkPlan): ConventionReq = ConventionFunc.create().conventionReqOf(plan) + def of(rowType: RowType, batchType: BatchType): ConventionReq = Impl(rowType, batchType) + + trait KnownChildrenConventions { + def requiredChildrenConventions(): Seq[ConventionReq] + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala index 9b745f94dfdc..73a126f8df27 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala @@ -29,7 +29,21 @@ import scala.collection.mutable * [[org.apache.gluten.extension.columnar.transition.Convention.BatchType]]'s definition. */ trait Transition { - def apply(plan: SparkPlan): SparkPlan + final def apply(plan: SparkPlan): SparkPlan = { + val out = apply0(plan) + if (out.fastEquals(plan)) { + assert( + this == Transition.empty, + "TransitionDef.empty / Transition.empty should be used when defining an empty transition.") + } + out + } + + final def isEmpty: Boolean = { + this == Transition.empty + } + + protected def apply0(plan: SparkPlan): SparkPlan } trait TransitionDef { @@ -53,12 +67,15 @@ object Transition { } private class ChainedTransition(first: Transition, second: Transition) extends Transition { - override def apply(plan: SparkPlan): SparkPlan = { + override def apply0(plan: SparkPlan): SparkPlan = { second(first(plan)) } } private def chain(first: Transition, second: Transition): Transition = { + if (first.isEmpty && second.isEmpty) { + return Transition.empty + } new ChainedTransition(first, second) } @@ -72,6 +89,15 @@ object Transition { } } + final def satisfies(conv: Convention, req: ConventionReq): Boolean = { + val none = new Transition { + override protected def apply0(plan: SparkPlan): SparkPlan = + throw new UnsupportedOperationException() + } + val transition = findTransition(conv, req)(none) + transition.isEmpty + } + protected def findTransition(from: Convention, to: ConventionReq)( orElse: => Transition): Transition private[transition] def update(): MutableFactory diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala index e0758cff7423..d02aadd493d4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transitions.scala @@ -17,16 +17,13 @@ package org.apache.gluten.extension.columnar.transition import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.execution.{SparkPlan, UnionExec} -import org.apache.spark.sql.execution.command.DataWritingCommandExec +import org.apache.spark.sql.execution.SparkPlan import scala.annotation.tailrec case class InsertTransitions(outputsColumnar: Boolean) extends Rule[SparkPlan] { - import InsertTransitions._ private val convFunc = ConventionFunc.create() override def apply(plan: SparkPlan): SparkPlan = { @@ -47,7 +44,7 @@ case class InsertTransitions(outputsColumnar: Boolean) extends Rule[SparkPlan] { if (node.children.isEmpty) { return node } - val convReq = childrenConvReqOf(node) + val convReq = convFunc.conventionReqOf(node) val newChildren = node.children.map { child => val from = convFunc.conventionOf(child) @@ -64,47 +61,6 @@ case class InsertTransitions(outputsColumnar: Boolean) extends Rule[SparkPlan] { } node.withNewChildren(newChildren) } - - private def childrenConvReqOf(node: SparkPlan): ConventionReq = node match { - // TODO: Consider C2C transitions as well when we have some. - case ColumnarToRowLike(_) | RowToColumnarLike(_) => - // C2R / R2C here since they are already removed by - // RemoveTransitions. - // It's current rule's mission to add C2Rs / R2Cs on demand. - throw new IllegalStateException("Unreachable code") - case write: DataWritingCommandExec if SparkShimLoader.getSparkShims.isPlannedV1Write(write) => - // To align with ApplyColumnarRulesAndInsertTransitions#insertTransitions - ConventionReq.any - case u: UnionExec => - // We force vanilla union to output row data to get best compatibility with vanilla Spark. - // As a result it's a common practice to rewrite it with GlutenPlan for offloading. - ConventionReq.of( - ConventionReq.RowType.Is(Convention.RowType.VanillaRow), - ConventionReq.BatchType.Any) - case other => - // In the normal case, children's convention should follow parent node's convention. - // Note, we don't have consider C2R / R2C here since they are already removed by - // RemoveTransitions. - val thisConv = convFunc.conventionOf(other) - thisConv.asReq() - } -} - -object InsertTransitions { - implicit private class ConventionOps(conv: Convention) { - def asReq(): ConventionReq = { - val rowTypeReq = conv.rowType match { - case Convention.RowType.None => ConventionReq.RowType.Any - case r => ConventionReq.RowType.Is(r) - } - - val batchTypeReq = conv.batchType match { - case Convention.BatchType.None => ConventionReq.BatchType.Any - case b => ConventionReq.BatchType.Is(b) - } - ConventionReq.of(rowTypeReq, batchTypeReq) - } - } } object RemoveTransitions extends Rule[SparkPlan] { diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala index f0ae4286f3d4..7417d9a5d729 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala @@ -16,15 +16,19 @@ */ package org.apache.gluten.planner.plan +import org.apache.gluten.extension.columnar.transition.{Convention, ConventionReq} +import org.apache.gluten.extension.columnar.transition.Convention.{KnownBatchType, KnownRowType} import org.apache.gluten.planner.metadata.GlutenMetadata -import org.apache.gluten.planner.property.{ConventionDef, Conventions} +import org.apache.gluten.planner.property.{Conv, ConvDef} import org.apache.gluten.ras.{Metadata, PlanModel} import org.apache.gluten.ras.property.PropertySet +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.{LeafExecNode, SparkPlan} +import org.apache.spark.sql.execution.{ColumnarToRowExec, LeafExecNode, SparkPlan} +import org.apache.spark.util.{SparkTaskUtil, TaskResources} import java.util.Objects @@ -36,25 +40,61 @@ object GlutenPlanModel { case class GroupLeafExec( groupId: Int, metadata: GlutenMetadata, - propertySet: PropertySet[SparkPlan]) - extends LeafExecNode { + constraintSet: PropertySet[SparkPlan]) + extends LeafExecNode + with KnownBatchType + with KnownRowType { + private val req: Conv.Req = constraintSet.get(ConvDef).asInstanceOf[Conv.Req] + override protected def doExecute(): RDD[InternalRow] = throw new IllegalStateException() override def output: Seq[Attribute] = metadata.schema().output - override def supportsColumnar: Boolean = - propertySet.get(ConventionDef) match { - case Conventions.ROW_BASED => false - case Conventions.VANILLA_COLUMNAR => true - case Conventions.GLUTEN_COLUMNAR => true - case Conventions.ANY => true + + override def supportsColumnar(): Boolean = { + batchType != Convention.BatchType.None + } + + override val batchType: Convention.BatchType = { + val out = req.req.requiredBatchType match { + case ConventionReq.BatchType.Any => Convention.BatchType.None + case ConventionReq.BatchType.Is(b) => b + } + out + } + + override val rowType: Convention.RowType = { + val out = req.req.requiredRowType match { + case ConventionReq.RowType.Any => Convention.RowType.None + case ConventionReq.RowType.Is(r) => r } + out + } } private object PlanModelImpl extends PlanModel[SparkPlan] { + private val fakeTc = SparkShimLoader.getSparkShims.createTestTaskContext() + private def fakeTc[T](body: => T): T = { + assert(!TaskResources.inSparkTask()) + SparkTaskUtil.setTaskContext(fakeTc) + try { + body + } finally { + SparkTaskUtil.unsetTaskContext() + } + } + override def childrenOf(node: SparkPlan): Seq[SparkPlan] = node.children - override def withNewChildren(node: SparkPlan, children: Seq[SparkPlan]): SparkPlan = { - node.withNewChildren(children) - } + override def withNewChildren(node: SparkPlan, children: Seq[SparkPlan]): SparkPlan = + node match { + case c2r: ColumnarToRowExec => + // Workaround: To bypass the assertion in ColumnarToRowExec's code if child is + // a group leaf. + fakeTc { + c2r.withNewChildren(children) + } + case other => + other.withNewChildren(children) + } override def hashCode(node: SparkPlan): Int = Objects.hashCode(node) @@ -63,8 +103,8 @@ object GlutenPlanModel { override def newGroupLeaf( groupId: Int, metadata: Metadata, - propSet: PropertySet[SparkPlan]): SparkPlan = - GroupLeafExec(groupId, metadata.asInstanceOf[GlutenMetadata], propSet) + constraintSet: PropertySet[SparkPlan]): SparkPlan = + GroupLeafExec(groupId, metadata.asInstanceOf[GlutenMetadata], constraintSet) override def isGroupLeaf(node: SparkPlan): Boolean = node match { case _: GroupLeafExec => true diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala new file mode 100644 index 000000000000..475f6292094c --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.planner.property + +import org.apache.gluten.extension.columnar.transition.{Convention, ConventionReq, Transition} +import org.apache.gluten.ras.{Property, PropertyDef} +import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} + +import org.apache.spark.sql.execution._ + +sealed trait Conv extends Property[SparkPlan] { + import Conv._ + override def definition(): PropertyDef[SparkPlan, _ <: Property[SparkPlan]] = { + ConvDef + } + + override def satisfies(other: Property[SparkPlan]): Boolean = { + val req = other.asInstanceOf[Req] + if (req.isAny) { + return true + } + val prop = this.asInstanceOf[Prop] + val out = Transition.factory.satisfies(prop.prop, req.req) + out + } +} + +object Conv { + val any: Conv = Req(ConventionReq.any) + + def of(conv: Convention): Conv = Prop(conv) + def req(req: ConventionReq): Conv = Req(req) + + def get(plan: SparkPlan): Conv = { + Conv.of(Convention.get(plan)) + } + + def findTransition(from: Conv, to: Conv): Transition = { + val prop = from.asInstanceOf[Prop] + val req = to.asInstanceOf[Req] + val out = Transition.factory.findTransition(prop.prop, req.req, new IllegalStateException()) + out + } + + case class Prop(prop: Convention) extends Conv + case class Req(req: ConventionReq) extends Conv { + def isAny: Boolean = { + req.requiredBatchType == ConventionReq.BatchType.Any && + req.requiredRowType == ConventionReq.RowType.Any + } + } +} + +object ConvDef extends PropertyDef[SparkPlan, Conv] { + // TODO: Should the convention-transparent ops (e.g., aqe shuffle read) support + // convention-propagation. Probably need to refactor getChildrenPropertyRequirements. + override def getProperty(plan: SparkPlan): Conv = { + conventionOf(plan) + } + + private def conventionOf(plan: SparkPlan): Conv = { + val out = Conv.get(plan) + out + } + + override def getChildrenConstraints( + constraint: Property[SparkPlan], + plan: SparkPlan): Seq[Conv] = { + val out = List.tabulate(plan.children.size)(_ => Conv.req(ConventionReq.get(plan))) + out + } + + override def any(): Conv = Conv.any +} + +case class ConvEnforcerRule(reqConv: Conv) extends RasRule[SparkPlan] { + override def shift(node: SparkPlan): Iterable[SparkPlan] = { + if (node.output.isEmpty) { + // Disable transitions for node that has output with empty schema. + return List.empty + } + val conv = Conv.get(node) + if (conv.satisfies(reqConv)) { + return List.empty + } + val transition = Conv.findTransition(conv, reqConv) + val after = transition.apply(node) + List(after) + } + + override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala deleted file mode 100644 index 5fe96ab79887..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.planner.property - -import org.apache.gluten.execution.RowToColumnarExecBase -import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike, Transitions} -import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec -import org.apache.gluten.ras.{Property, PropertyDef} -import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} -import org.apache.gluten.sql.shims.SparkShimLoader -import org.apache.gluten.utils.PlanUtil - -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, QueryStageExec} -import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec -import org.apache.spark.sql.execution.exchange.ReusedExchangeExec - -sealed trait Convention extends Property[SparkPlan] { - override def definition(): PropertyDef[SparkPlan, _ <: Property[SparkPlan]] = { - ConventionDef - } - - override def satisfies(other: Property[SparkPlan]): Boolean = other match { - case Conventions.ANY => true - case c: Convention => c == this - case _ => throw new IllegalStateException() - } -} - -object Conventions { - // FIXME: Velox and CH should have different conventions? - case object ROW_BASED extends Convention - case object VANILLA_COLUMNAR extends Convention - case object GLUTEN_COLUMNAR extends Convention - case object ANY extends Convention -} - -object ConventionDef extends PropertyDef[SparkPlan, Convention] { - // TODO: Should the convention-transparent ops (e.g., aqe shuffle read) support - // convention-propagation. Probably need to refactor getChildrenPropertyRequirements. - override def getProperty(plan: SparkPlan): Convention = plan match { - case _: GroupLeafExec => throw new IllegalStateException() - case other => conventionOf(other) - } - - private def conventionOf(plan: SparkPlan): Convention = plan match { - case g: GroupLeafExec => g.propertySet.get(ConventionDef) - case ColumnarToRowExec(child) => Conventions.ROW_BASED - case RowToColumnarExec(child) => Conventions.VANILLA_COLUMNAR - case ColumnarToRowLike(child) => Conventions.ROW_BASED - case RowToColumnarLike(child) => Conventions.GLUTEN_COLUMNAR - case q: QueryStageExec => conventionOf(q.plan) - case r: ReusedExchangeExec => conventionOf(r.child) - case a: AdaptiveSparkPlanExec => conventionOf(a.executedPlan) - case i: InMemoryTableScanExec => getCacheConvention(i) - case p if canPropagateConvention(p) => - val childrenProps = p.children.map(conventionOf).distinct - assert(childrenProps.size == 1) - childrenProps.head - case _: GlutenPlan => Conventions.GLUTEN_COLUMNAR - case p if p.supportsColumnar => Conventions.VANILLA_COLUMNAR - case p if SparkShimLoader.getSparkShims.supportsRowBased(p) => Conventions.ROW_BASED - case other => throw new IllegalStateException(s"Unable to get convention of $other") - } - - override def getChildrenConstraints( - constraint: Property[SparkPlan], - plan: SparkPlan): Seq[Convention] = plan match { - case ColumnarToRowExec(child) => Seq(Conventions.VANILLA_COLUMNAR) - case ColumnarToRowLike(child) => Seq(Conventions.GLUTEN_COLUMNAR) - case RowToColumnarLike(child) => Seq(Conventions.ROW_BASED) - case p if canPropagateConvention(p) => - p.children.map(_ => constraint.asInstanceOf[Convention]) - case other => - val conv = conventionOf(other) - other.children.map(_ => conv) - } - - override def any(): Convention = Conventions.ANY - - private def canPropagateConvention(plan: SparkPlan): Boolean = plan match { - case p: AQEShuffleReadExec => true - case p: InputAdapter => true - case p: WholeStageCodegenExec => true - case _ => false - } - - private def getCacheConvention(i: InMemoryTableScanExec): Convention = { - if (PlanUtil.isGlutenTableCache(i)) { - Conventions.GLUTEN_COLUMNAR - } else if (i.supportsColumnar) { - Conventions.VANILLA_COLUMNAR - } else { - Conventions.ROW_BASED - } - } -} - -case class ConventionEnforcerRule(reqConv: Convention) extends RasRule[SparkPlan] { - override def shift(node: SparkPlan): Iterable[SparkPlan] = { - if (node.output.isEmpty) { - // Disable transitions for node that has output with empty schema. - return List.empty - } - val conv = ConventionDef.getProperty(node) - if (conv.satisfies(reqConv)) { - return List.empty - } - (conv, reqConv) match { - case (Conventions.VANILLA_COLUMNAR, Conventions.ROW_BASED) => - List(ColumnarToRowExec(node)) - case (Conventions.ROW_BASED, Conventions.VANILLA_COLUMNAR) => - List(RowToColumnarExec(node)) - case (Conventions.GLUTEN_COLUMNAR, Conventions.ROW_BASED) => - List(Transitions.toRowPlan(node)) - case (Conventions.ROW_BASED, Conventions.GLUTEN_COLUMNAR) => - val attempt = Transitions.toBackendBatchPlan(node) - if (attempt.asInstanceOf[RowToColumnarExecBase].doValidate().isValid) { - List(attempt) - } else { - List.empty - } - case (Conventions.VANILLA_COLUMNAR, Conventions.GLUTEN_COLUMNAR) => - List(Transitions.toBackendBatchPlan(ColumnarToRowExec(node))) - case (Conventions.GLUTEN_COLUMNAR, Conventions.VANILLA_COLUMNAR) => - List(RowToColumnarExec(Transitions.toRowPlan(node))) - case _ => List.empty - } - } - - override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/property/GlutenPropertyModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/property/GlutenPropertyModel.scala index a998c935d198..115ab4471fb6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/property/GlutenPropertyModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/property/GlutenPropertyModel.scala @@ -28,14 +28,14 @@ object GlutenPropertyModel { private object PropertyModelImpl extends PropertyModel[SparkPlan] { override def propertyDefs: Seq[PropertyDef[SparkPlan, _ <: Property[SparkPlan]]] = - Seq(ConventionDef) + Seq(ConvDef) override def newEnforcerRuleFactory( propertyDef: PropertyDef[SparkPlan, _ <: Property[SparkPlan]]) : EnforcerRuleFactory[SparkPlan] = (reqProp: Property[SparkPlan]) => { propertyDef match { - case ConventionDef => - Seq(ConventionEnforcerRule(reqProp.asInstanceOf[Convention])) + case ConvDef => + Seq(ConvEnforcerRule(reqProp.asInstanceOf[Conv])) } } } diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala new file mode 100644 index 000000000000..92a12b3c6e87 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkTaskUtil.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.util + +import org.apache.spark.TaskContext + +object SparkTaskUtil { + def setTaskContext(taskContext: TaskContext): Unit = { + TaskContext.setTaskContext(taskContext) + } + + def unsetTaskContext(): Unit = { + TaskContext.unset() + } +} diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala index 3f40793d9da5..58a88e1f49ce 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ArrowBatch.scala @@ -17,7 +17,8 @@ package org.apache.gluten.columnarbatch -import org.apache.gluten.extension.columnar.transition.Convention +import org.apache.gluten.extension.columnar.transition.{Convention, TransitionDef} +import org.apache.gluten.extension.columnar.transition.Convention.BatchType.VanillaBatch import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan} @@ -38,4 +39,8 @@ object ArrowBatch extends Convention.BatchType { (plan: SparkPlan) => { ColumnarToRowExec(plan) }) + + // Arrow batch is one-way compatible with vanilla batch since it provides valid + // #get(...) implementations. + toBatch(VanillaBatch, TransitionDef.empty) } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/PlanModel.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/PlanModel.scala index 34924ccbf078..bac9d0b6465c 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/PlanModel.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/PlanModel.scala @@ -26,7 +26,7 @@ trait PlanModel[T <: AnyRef] { def equals(one: T, other: T): Boolean // Group operations. - def newGroupLeaf(groupId: Int, meta: Metadata, propSet: PropertySet[T]): T + def newGroupLeaf(groupId: Int, meta: Metadata, constraintSet: PropertySet[T]): T def isGroupLeaf(node: T): Boolean def getGroupId(node: T): Int } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala index 804d04d814e5..f705a2901be9 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala @@ -30,9 +30,7 @@ trait Optimization[T <: AnyRef] { plan: T, constraintSet: PropertySet[T], altConstraintSets: Seq[PropertySet[T]]): RasPlanner[T] - - def propSetOf(plan: T): PropertySet[T] - + def anyPropSet(): PropertySet[T] def withNewConfig(confFunc: RasConfig => RasConfig): Optimization[T] } @@ -49,7 +47,7 @@ object Optimization { implicit class OptimizationImplicits[T <: AnyRef](opt: Optimization[T]) { def newPlanner(plan: T): RasPlanner[T] = { - opt.newPlanner(plan, opt.propSetOf(plan), List.empty) + opt.newPlanner(plan, opt.anyPropSet(), List.empty) } def newPlanner(plan: T, constraintSet: PropertySet[T]): RasPlanner[T] = { opt.newPlanner(plan, constraintSet, List.empty) @@ -113,15 +111,6 @@ class Ras[T <: AnyRef] private ( // Node groups don't have user-defined cost, expect exception here. metadataModel.metadataOf(dummyGroup) } - propertyModel.propertyDefs.foreach { - propDef => - // Node groups don't have user-defined property, expect exception here. - assertThrows( - "Group is not allowed to return its property directly to optimizer (optimizer already" + - " knew that). It's expected to throw an exception when getting its property but not") { - propDef.getProperty(dummyGroup) - } - } } override def newPlanner( @@ -131,7 +120,12 @@ class Ras[T <: AnyRef] private ( RasPlanner(this, altConstraintSets, constraintSet, plan) } - override def propSetOf(plan: T): PropertySet[T] = propertySetFactory().get(plan) + override def anyPropSet(): PropertySet[T] = propertySetFactory().any() + + private[ras] def propSetOf(plan: T): PropertySet[T] = { + val out = propertySetFactory().get(plan) + out + } private[ras] def withNewChildren(node: T, newChildren: Seq[T]): T = { val oldChildren = planModel.childrenOf(node) diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasGroup.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasGroup.scala index b5e9c98912f4..9591fbb22554 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasGroup.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasGroup.scala @@ -22,7 +22,7 @@ import org.apache.gluten.ras.property.PropertySet trait RasGroup[T <: AnyRef] { def id(): Int def clusterKey(): RasClusterKey - def propSet(): PropertySet[T] + def constraintSet(): PropertySet[T] def self(): T def nodes(store: MemoStore[T]): Iterable[CanonicalNode[T]] } @@ -40,17 +40,17 @@ object RasGroup { ras: Ras[T], clusterKey: RasClusterKey, override val id: Int, - override val propSet: PropertySet[T]) + override val constraintSet: PropertySet[T]) extends RasGroup[T] { - private val groupLeaf: T = ras.planModel.newGroupLeaf(id, clusterKey.metadata, propSet) + private val groupLeaf: T = ras.planModel.newGroupLeaf(id, clusterKey.metadata, constraintSet) override def clusterKey(): RasClusterKey = clusterKey override def self(): T = groupLeaf override def nodes(store: MemoStore[T]): Iterable[CanonicalNode[T]] = { - store.getCluster(clusterKey).nodes().filter(n => n.propSet().satisfies(propSet)) + store.getCluster(clusterKey).nodes().filter(n => n.propSet().satisfies(constraintSet)) } override def toString(): String = { - s"RasGroup(id=$id, clusterKey=$clusterKey, propSet=$propSet))" + s"RasGroup(id=$id, clusterKey=$clusterKey, constraintSet=$constraintSet))" } } } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasNode.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasNode.scala index 710a4e682293..8c9b52605678 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasNode.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasNode.scala @@ -95,7 +95,11 @@ trait GroupNode[T <: AnyRef] extends RasNode[T] { object GroupNode { def apply[T <: AnyRef](ras: Ras[T], group: RasGroup[T]): GroupNode[T] = { - new GroupNodeImpl[T](ras, group.self(), group.propSet(), group.id()) + val self = group.self() + // Re-derive property set of group leaf. User should define an appropriate conversion + // from group constraints to its output properties in property model or plan model. + val propSet = ras.propSetOf(self) + new GroupNodeImpl[T](ras, self, propSet, group.id()) } private class GroupNodeImpl[T <: AnyRef]( diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala index 3db649b6457c..c4d3e4881c43 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala @@ -130,7 +130,7 @@ object ExhaustivePlanner { private def applyEnforcerRules(): Unit = { allGroups.foreach { group => - val constraintSet = group.propSet() + val constraintSet = group.constraintSet() val enforcerRules = enforcerRuleSet.rulesOf(constraintSet) if (enforcerRules.nonEmpty) { val shapes = enforcerRules.map(_.shape()) diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/RuleApplier.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/RuleApplier.scala index 6b4082c7eb15..3d94a99967db 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/RuleApplier.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/RuleApplier.scala @@ -64,7 +64,7 @@ object RuleApplier { equiv => closure .openFor(cKey) - .memorize(equiv, ras.propertySetFactory().get(equiv)) + .memorize(equiv, ras.anyPropSet()) } } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/vis/GraphvizVisualizer.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/vis/GraphvizVisualizer.scala index b420d8c2978a..d7d14cf3a77f 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/vis/GraphvizVisualizer.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/vis/GraphvizVisualizer.scala @@ -148,7 +148,7 @@ class GraphvizVisualizer[T <: AnyRef](ras: Ras[T], memoState: MemoState[T], best } private def describeGroupVerbose(group: RasGroup[T]): String = { - s"[Group ${group.id()}: ${group.propSet().getMap.values.toIndexedSeq}]" + s"[Group ${group.id()}: ${group.constraintSet().getMap.values.toIndexedSeq}]" } private def describeNode( diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala index f1c319873355..60ec2eedd410 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala @@ -411,9 +411,12 @@ object OperationSuite { equalsCount += 1 delegated.equals(one, other) } - override def newGroupLeaf(groupId: Int, metadata: Metadata, propSet: PropertySet[T]): T = { + override def newGroupLeaf( + groupId: Int, + metadata: Metadata, + constraintSet: PropertySet[T]): T = { newGroupLeafCount += 1 - delegated.newGroupLeaf(groupId, metadata, propSet) + delegated.newGroupLeaf(groupId, metadata, constraintSet) } override def isGroupLeaf(node: T): Boolean = { isGroupLeafCount += 1 diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala index aed032226819..eb4babe069e5 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala @@ -72,7 +72,7 @@ abstract class PropertySuite extends AnyFunSuite { memo.memorize(ras, PassNodeType(1, PassNodeType(1, PassNodeType(1, TypedLeaf(TypeB, 1))))) val state = memo.newState() assert(state.allClusters().size == 4) - assert(state.getGroupCount() == 8) + assert(state.getGroupCount() == 4) } test(s"Get property") { @@ -573,7 +573,7 @@ object PropertySuite { override def any(): DummyProperty = DummyProperty(Int.MinValue) override def getProperty(plan: TestNode): DummyProperty = { plan match { - case Group(_, _, _) => throw new IllegalStateException() + case g: Group => g.constraintSet.get(this) case PUnary(_, prop, _) => prop case PLeaf(_, prop) => prop case PBinary(_, prop, _, _) => prop @@ -645,7 +645,7 @@ object PropertySuite { case class PassNodeType(override val selfCost: Long, child: TestNode) extends TypedNode { override def nodeType: NodeType = child match { case n: TypedNode => n.nodeType - case g: Group => g.propSet.get(NodeTypeDef) + case g: Group => g.constraintSet.get(NodeTypeDef) case _ => throw new IllegalStateException() } @@ -669,7 +669,7 @@ object PropertySuite { override def shift(node: TestNode): Iterable[TestNode] = { node match { case group: Group => - val groupType = group.propSet.get(NodeTypeDef) + val groupType = group.constraintSet.get(NodeTypeDef) if (groupType.satisfies(reqType)) { List(group) } else { @@ -710,6 +710,7 @@ object PropertySuite { object NodeTypeDef extends PropertyDef[TestNode, NodeType] { override def getProperty(plan: TestNode): NodeType = plan match { + case g: Group => g.constraintSet.get(this) case typed: TypedNode => typed.nodeType case _ => throw new IllegalStateException() } diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/RasSuiteBase.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/RasSuiteBase.scala index b5455d6afae4..65c4d5a073a4 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/RasSuiteBase.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/RasSuiteBase.scala @@ -49,7 +49,7 @@ object RasSuiteBase { def withNewChildren(children: Seq[TestNode]): TestNode = this } - case class Group(id: Int, meta: Metadata, propSet: PropertySet[TestNode]) extends LeafLike { + case class Group(id: Int, meta: Metadata, constraintSet: PropertySet[TestNode]) extends LeafLike { override def selfCost(): Long = Long.MaxValue override def makeCopy(): LeafLike = copy() } @@ -113,8 +113,8 @@ object RasSuiteBase { override def newGroupLeaf( groupId: Int, meta: Metadata, - propSet: PropertySet[TestNode]): TestNode = - Group(groupId, meta, propSet) + constraintSet: PropertySet[TestNode]): TestNode = + Group(groupId, meta, constraintSet) override def getGroupId(node: TestNode): Int = node match { case ngl: Group => ngl.id @@ -163,7 +163,7 @@ object RasSuiteBase { implicit class MemoLikeImplicits[T <: AnyRef](val memo: MemoLike[T]) { def memorize(ras: Ras[T], node: T): RasGroup[T] = { - memo.memorize(node, ras.propSetOf(node)) + memo.memorize(node, ras.anyPropSet()) } } diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala index 7bb713afe821..37d66e2bd703 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala @@ -121,7 +121,7 @@ object MockMemoState { class MockMutableGroup[T <: AnyRef] private ( override val id: Int, override val clusterKey: RasClusterKey, - override val propSet: PropertySet[T], + override val constraintSet: PropertySet[T], override val self: T) extends RasGroup[T] { private val nodes: mutable.ArrayBuffer[CanonicalNode[T]] = mutable.ArrayBuffer() diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockRasPath.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockRasPath.scala index cd8050e5f8f3..bf267a4b68ec 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockRasPath.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockRasPath.scala @@ -27,7 +27,7 @@ object MockRasPath { def mock[T <: AnyRef](ras: Ras[T], node: T, keys: PathKeySet): RasPath[T] = { val memo = Memo(ras) - val g = memo.memorize(node, ras.propSetOf(node)) + val g = memo.memorize(node, ras.anyPropSet()) val state = memo.newState() val groupSupplier = state.asGroupSupplier() assert(g.nodes(state).size == 1) diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/specific/DistributedSuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/specific/DistributedSuite.scala index 2aefc54e90a5..e930e4da221c 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/specific/DistributedSuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/specific/DistributedSuite.scala @@ -251,6 +251,7 @@ object DistributedSuite { private object DistributionDef extends PropertyDef[TestNode, Distribution] { override def getProperty(plan: TestNode): Distribution = plan match { + case g: Group => g.constraintSet.get(this) case d: DNode => d.getDistribution() case _ => throw new UnsupportedOperationException() @@ -308,6 +309,7 @@ object DistributedSuite { // FIXME: Handle non-ordering as well as non-distribution private object OrderingDef extends PropertyDef[TestNode, Ordering] { override def getProperty(plan: TestNode): Ordering = plan match { + case g: Group => g.constraintSet.get(this) case d: DNode => d.getOrdering() case _ => throw new UnsupportedOperationException() } @@ -383,7 +385,7 @@ object DistributedSuite { with UnaryLike { override def getDistribution(): Distribution = { val childDistribution = child match { - case g: Group => g.propSet.get(DistributionDef) + case g: Group => g.constraintSet.get(DistributionDef) case other => DistributionDef.getProperty(other) } if (childDistribution == NoneDistribution) { @@ -415,7 +417,7 @@ object DistributedSuite { extends DNode with UnaryLike { override def getDistribution(): Distribution = child match { - case g: Group => g.propSet.get(DistributionDef) + case g: Group => g.constraintSet.get(DistributionDef) case other => DistributionDef.getProperty(other) } @@ -433,7 +435,7 @@ object DistributedSuite { with UnaryLike { override def getDistribution(): Distribution = { val childDistribution = child match { - case g: Group => g.propSet.get(DistributionDef) + case g: Group => g.constraintSet.get(DistributionDef) case other => DistributionDef.getProperty(other) } if (childDistribution == NoneDistribution) { @@ -463,12 +465,12 @@ object DistributedSuite { case class DProject(override val child: TestNode) extends DNode with UnaryLike { override def getDistribution(): Distribution = child match { - case g: Group => g.propSet.get(DistributionDef) + case g: Group => g.constraintSet.get(DistributionDef) case other => DistributionDef.getProperty(other) } override def getDistributionConstraints(req: Distribution): Seq[Distribution] = List(req) override def getOrdering(): Ordering = child match { - case g: Group => g.propSet.get(OrderingDef) + case g: Group => g.constraintSet.get(OrderingDef) case other => OrderingDef.getProperty(other) } override def getOrderingConstraints(req: Ordering): Seq[Ordering] = List(req) @@ -482,7 +484,7 @@ object DistributedSuite { with UnaryLike { override def getDistribution(): Distribution = { val childDistribution = child match { - case g: Group => g.propSet.get(DistributionDef) + case g: Group => g.constraintSet.get(DistributionDef) case other => DistributionDef.getProperty(other) } if (childDistribution == NoneDistribution) { @@ -501,13 +503,13 @@ object DistributedSuite { case class DSort(keys: Seq[String], override val child: TestNode) extends DNode with UnaryLike { override def getDistribution(): Distribution = child match { - case g: Group => g.propSet.get(DistributionDef) + case g: Group => g.constraintSet.get(DistributionDef) case other => DistributionDef.getProperty(other) } override def getDistributionConstraints(req: Distribution): Seq[Distribution] = List(req) override def getOrdering(): Ordering = { val childOrdering = child match { - case g: Group => g.propSet.get(OrderingDef) + case g: Group => g.constraintSet.get(OrderingDef) case other => OrderingDef.getProperty(other) } if (childOrdering.satisfies(SimpleOrdering(keys))) { From a7e536ebb11a685381bf8a799f16f42789b7bc43 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 22 May 2024 18:59:31 +0800 Subject: [PATCH 129/402] [VL] Daily Update Velox Version (2024_05_22) (#5834) --- cpp/velox/benchmarks/PlanValidatorUtil.cc | 4 ++-- cpp/velox/compute/WholeStageResultIterator.cc | 2 +- cpp/velox/jni/VeloxJniWrapper.cc | 4 ++-- cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc | 2 +- ep/build-velox/src/get_velox.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/velox/benchmarks/PlanValidatorUtil.cc b/cpp/velox/benchmarks/PlanValidatorUtil.cc index e299b4620028..46f2733f29ea 100644 --- a/cpp/velox/benchmarks/PlanValidatorUtil.cc +++ b/cpp/velox/benchmarks/PlanValidatorUtil.cc @@ -45,9 +45,9 @@ int main(int argc, char** argv) { conf.insert({kDebugModeEnabled, "true"}); initVeloxBackend(conf); std::unordered_map configs{{core::QueryConfig::kSparkPartitionId, "0"}}; - core::QueryCtx queryCtx(nullptr, core::QueryConfig(configs)); + auto queryCtx = core::QueryCtx::create(nullptr, core::QueryConfig(configs)); auto pool = defaultLeafVeloxMemoryPool().get(); - core::ExecCtx execCtx(pool, &queryCtx); + core::ExecCtx execCtx(pool, queryCtx.get()); ::substrait::Plan subPlan; parseProtobuf(reinterpret_cast(plan.data()), plan.size(), &subPlan); diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 852c7e3cc277..f719c119c3e0 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -169,7 +169,7 @@ std::shared_ptr WholeStageResultIterator::createNewVeloxQ std::unordered_map> connectorConfigs; connectorConfigs[kHiveConnectorId] = createConnectorConfig(); - std::shared_ptr ctx = std::make_shared( + std::shared_ptr ctx = velox::core::QueryCtx::create( nullptr, facebook::velox::core::QueryConfig{getQueryContextConf()}, connectorConfigs, diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index 7884280c3c94..9da7355d1b3a 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -120,10 +120,10 @@ Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeValidateWithFail // A query context with dummy configs. Used for function validation. std::unordered_map configs{ {velox::core::QueryConfig::kSparkPartitionId, "0"}, {velox::core::QueryConfig::kSessionTimezone, "GMT"}}; - velox::core::QueryCtx queryCtx(nullptr, velox::core::QueryConfig(configs)); + auto queryCtx = velox::core::QueryCtx::create(nullptr, velox::core::QueryConfig(configs)); auto pool = gluten::defaultLeafVeloxMemoryPool().get(); // An execution context used for function validation. - velox::core::ExecCtx execCtx(pool, &queryCtx); + velox::core::ExecCtx execCtx(pool, queryCtx.get()); gluten::SubstraitToVeloxPlanValidator planValidator(pool, &execCtx); jclass infoCls = env->FindClass("Lorg/apache/gluten/validate/NativePlanValidationInfo;"); diff --git a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc index d5eafa1e2b2f..0a957f038f8d 100644 --- a/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc +++ b/cpp/velox/tests/Substrait2VeloxPlanValidatorTest.cc @@ -46,7 +46,7 @@ class Substrait2VeloxPlanValidatorTest : public exec::test::HiveConnectorTestBas } bool validatePlan(::substrait::Plan& plan) { - std::shared_ptr queryCtx = std::make_shared(); + auto queryCtx = core::QueryCtx::create(); // An execution context used for function validation. std::unique_ptr execCtx = std::make_unique(pool_.get(), queryCtx.get()); diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 70b3a9b09987..fbb0f706742f 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_21 +VELOX_BRANCH=2024_05_22 VELOX_HOME="" #Set on run gluten on HDFS From a41ef6ac496ddece8a2774cedd6dfcb8053e9009 Mon Sep 17 00:00:00 2001 From: James Xu Date: Wed, 22 May 2024 23:30:17 +0800 Subject: [PATCH 130/402] [GLUTEN-5832][VL] Fix build on macOS (#5833) --- cpp/core/memory/AllocationListener.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/core/memory/AllocationListener.h b/cpp/core/memory/AllocationListener.h index a3c0a72cbc30..d43a621de9a0 100644 --- a/cpp/core/memory/AllocationListener.h +++ b/cpp/core/memory/AllocationListener.h @@ -17,7 +17,7 @@ #pragma once -#include +#include #include namespace gluten { From b798ef5f1bab3a6674d3c69473b5ec781ffe2ca3 Mon Sep 17 00:00:00 2001 From: sharmaplkt Date: Thu, 23 May 2024 04:51:02 +0530 Subject: [PATCH 131/402] [GLUTEN-5844][CORE] Refactor the usage of spark.gluten.enabled (#5845) [GLUTEN-5844][CORE] Refactor the usage of spark.gluten.enabled. Co-authored-by: Pulkit Sharma --- ...utenClickHouseMergeTreeOptimizeSuite.scala | 50 ++++++++++--------- ...lutenClickHouseNativeWriteTableSuite.scala | 20 ++++---- ...seTPCHColumnarShuffleParquetAQESuite.scala | 3 +- ...enClickHouseTPCHSaltNullParquetSuite.scala | 3 +- .../GlutenClickHouseTableAfterRestart.scala | 6 ++- .../GlutenClickhouseFunctionSuite.scala | 2 +- .../org/apache/gluten/s3/S3AuthSuite.scala | 2 +- .../benchmarks/CHAggAndShuffleBenchmark.scala | 3 +- .../benchmarks/CHParquetReadBenchmark.scala | 3 +- .../WholeStageTransformerSuite.scala | 3 +- .../benchmarks/ParquetReadBenchmark.scala | 2 +- .../GlutenCustomerExtensionSuite.scala | 4 +- .../benchmarks/ParquetReadBenchmark.scala | 2 +- .../GlutenCustomerExtensionSuite.scala | 4 +- .../benchmarks/ParquetReadBenchmark.scala | 2 +- .../GlutenCustomerExtensionSuite.scala | 4 +- .../benchmarks/ParquetReadBenchmark.scala | 2 +- .../GlutenCustomerExtensionSuite.scala | 4 +- 18 files changed, 68 insertions(+), 51 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index ae0cd170d3fa..d4302193f5b8 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.gluten.execution +import org.apache.gluten.GlutenConfig + import org.apache.spark.SparkConf import org.apache.spark.sql.SaveMode import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper @@ -122,11 +124,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) spark.sql("VACUUM lineitem_mergetree_optimize_p RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() assert(ret2.apply(0).get(0) == 600572) @@ -154,14 +156,14 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 812) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 232) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") // the second VACUUM will remove some empty folders assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 220) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() assert(ret2.apply(0).get(0) == 600572) @@ -185,13 +187,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 398) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 286) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 270) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() assert(ret2.apply(0).get(0) == 600572) @@ -216,13 +218,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 398) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 286) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 270) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() assert(ret2.apply(0).get(0) == 600572) @@ -246,11 +248,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p5") - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 99) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() assert(ret.apply(0).get(0) == 600572) @@ -266,11 +268,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p5") - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() assert(ret.apply(0).get(0) == 600572) @@ -279,11 +281,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite // now merge all parts (testing merging from merged parts) spark.sql("optimize lineitem_mergetree_optimize_p5") - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 77) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() assert(ret.apply(0).get(0) == 600572) @@ -309,7 +311,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p6").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { if (sparkVersion.equals("3.2")) 931 else 1014 }) @@ -318,7 +320,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { if (sparkVersion.equals("3.2")) 439 else 445 }) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p6").collect() assert(ret2.apply(0).get(0) == 600572) @@ -341,9 +343,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite |""".stripMargin) spark.sql("optimize lineitem_mergetree_index") - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("vacuum lineitem_mergetree_index") - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val df = spark .sql(s""" @@ -387,10 +389,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") assert(countFiles(new File(dataPath)) == 99) val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() @@ -408,10 +410,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") assert(countFiles(new File(dataPath)) == 93) val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() @@ -422,10 +424,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") assert(countFiles(new File(dataPath)) == 77) val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala index 6b59242972d1..9269303d9251 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala @@ -173,7 +173,7 @@ class GlutenClickHouseNativeWriteTableSuite test("test insert into dir") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val originDF = spark.createDataFrame(genTestData()) originDF.createOrReplaceTempView("origin_table") @@ -209,7 +209,7 @@ class GlutenClickHouseNativeWriteTableSuite withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), ("spark.sql.orc.compression.codec", "lz4"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val originDF = spark.createDataFrame(genTestData()) originDF.createOrReplaceTempView("origin_table") @@ -260,7 +260,7 @@ class GlutenClickHouseNativeWriteTableSuite test("test CTAS") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val originDF = spark.createDataFrame(genTestData()) originDF.createOrReplaceTempView("origin_table") @@ -311,7 +311,7 @@ class GlutenClickHouseNativeWriteTableSuite ("spark.gluten.sql.native.writer.enabled", "true"), ("spark.sql.hive.convertMetastoreParquet", "false"), ("spark.sql.hive.convertMetastoreOrc", "false"), - ("spark.gluten.enabled", "true") + (GlutenConfig.GLUTEN_ENABLED.key, "true") ) { val originDF = spark.createDataFrame(genTestData()) @@ -429,7 +429,7 @@ class GlutenClickHouseNativeWriteTableSuite test("test 2-col partitioned table") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val fields: ListMap[String, String] = ListMap( ("string_field", "string"), @@ -467,7 +467,7 @@ class GlutenClickHouseNativeWriteTableSuite " ignore because takes too long") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val fields: ListMap[String, String] = ListMap( ("date_field", "date"), @@ -508,7 +508,7 @@ class GlutenClickHouseNativeWriteTableSuite ignore("test hive parquet/orc table, all columns being partitioned. ") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val fields: ListMap[String, String] = ListMap( ("date_field", "date"), @@ -547,7 +547,7 @@ class GlutenClickHouseNativeWriteTableSuite test(("test hive parquet/orc table with aggregated results")) { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val fields: ListMap[String, String] = ListMap( ("sum(int_field)", "bigint") @@ -573,7 +573,7 @@ class GlutenClickHouseNativeWriteTableSuite test("test 1-col partitioned + 1-col bucketed table") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { val fields: ListMap[String, String] = ListMap( ("string_field", "string"), @@ -911,7 +911,7 @@ class GlutenClickHouseNativeWriteTableSuite test("GLUTEN-4316: fix crash on dynamic partition inserting") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { formats.foreach( format => { val tbl = "t_" + format diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala index 1b3929dcc845..6caac99181fa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala @@ -16,6 +16,7 @@ */ package org.apache.gluten.execution +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.GlutenPlan import org.apache.spark.SparkConf @@ -298,7 +299,7 @@ class GlutenClickHouseTPCHColumnarShuffleParquetAQESuite } test("Test 'spark.gluten.enabled' false") { - withSQLConf(("spark.gluten.enabled", "false")) { + withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) { runTPCHQuery(2, noFallBack = false) { df => val glutenPlans = collect(df.queryExecution.executedPlan) { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index a1bba300ed22..eec0ad874c5d 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -16,6 +16,7 @@ */ package org.apache.gluten.execution +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.GlutenPlan import org.apache.spark.{SparkConf, SparkException} @@ -1231,7 +1232,7 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr } test("Test 'spark.gluten.enabled' false") { - withSQLConf(("spark.gluten.enabled", "false")) { + withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) { runTPCHQuery(2, noFallBack = false) { df => val glutenPlans = df.queryExecution.executedPlan.collect { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala index a673d4ba3bb4..9e55df0fa836 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala @@ -16,6 +16,8 @@ */ package org.apache.gluten.execution +import org.apache.gluten.GlutenConfig + import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.SparkSession.{getActiveSession, getDefaultSession} @@ -248,9 +250,9 @@ class GlutenClickHouseTableAfterRestart restartSpark() - spark.sql("set spark.gluten.enabled=false") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("vacuum table_restart_vacuum") - spark.sql("set spark.gluten.enabled=true") + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") assert(spark.sql("select count(*) from table_restart_vacuum").collect().apply(0).get(0) == 4) } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala index c90e9131a5a4..63e105e12a72 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala @@ -121,7 +121,7 @@ class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite { test("test uuid - write and read") { withSQLConf( ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.gluten.enabled", "true")) { + (GlutenConfig.GLUTEN_ENABLED.key, "true")) { spark.sql("drop table if exists uuid_test") spark.sql("create table if not exists uuid_test (id string) stored as parquet") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala index b2ac3672a1e1..32c64c78a69a 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala @@ -102,7 +102,7 @@ class S3AuthSuite extends AnyFunSuite { } def withGluten(enable: Boolean): Builder = { - builder.config("spark.gluten.enabled", enable.toString) + builder.config(GlutenConfig.GLUTEN_ENABLED.key, enable.toString) } } diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala index 19ced951795b..358b785187a0 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks +import org.apache.gluten.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, ProjectExecTransformer, WholeStageTransformer} import org.apache.gluten.sql.shims.SparkShimLoader @@ -303,7 +304,7 @@ object CHAggAndShuffleBenchmark extends SqlBasedBenchmark with CHSqlBasedBenchma // Get the file partitions for generating the `FileScanRDD` val filePartitions = fileScan.getPartitions .map(_.asInstanceOf[FilePartition]) - spark.conf.set("spark.gluten.enabled", "false") + spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") val sparkExecutedPlan = allStages.queryExecution.executedPlan // Get the `FileSourceScanExec` diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala index 1cc8c83835e2..5e802eeed1e4 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks +import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformContext} import org.apache.gluten.expression.ConverterUtils @@ -189,7 +190,7 @@ object CHParquetReadBenchmark extends SqlBasedBenchmark with CHSqlBasedBenchmark } if (executedVanilla) { - spark.conf.set("spark.gluten.enabled", "false") + spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") val vanillaParquet = spark.sql(s""" |select $scanSchema from parquet.`$parquetDir` diff --git a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index 8e8743857139..5f60de27b415 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -16,6 +16,7 @@ */ package org.apache.gluten.execution +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.GlutenPlan import org.apache.gluten.test.FallbackUtil import org.apache.gluten.utils.Arm @@ -320,7 +321,7 @@ abstract class WholeStageTransformerSuite noFallBack) protected def vanillaSparkConfs(): Seq[(String, String)] = { - List(("spark.gluten.enabled", "false")) + List((GlutenConfig.GLUTEN_ENABLED.key, "false")) } protected def checkDataFrame( diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index ad08318bbde5..ef96994582c4 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -165,7 +165,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { } if (executedVanilla) { - spark.conf.set("spark.gluten.enabled", "false") + spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") val vanillaParquet = spark.sql(s""" |select $scanSchema from parquet.`$parquetDir` diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index 18a9f5c9fc8c..5e6c66265ad7 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.extension +import org.apache.gluten.GlutenConfig + import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait @@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait { } testGluten("test customer column rules") { - withSQLConf(("spark.gluten.enabled", "false")) { + withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) { sql("create table my_parquet(id int) using parquet") sql("insert into my_parquet values (1)") sql("insert into my_parquet values (2)") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 7d8a292042e8..f61f7ebba9b2 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -165,7 +165,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { } if (executedVanilla) { - spark.conf.set("spark.gluten.enabled", "false") + spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") val vanillaParquet = spark.sql(s""" |select $scanSchema from parquet.`$parquetDir` diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index 18a9f5c9fc8c..5e6c66265ad7 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.extension +import org.apache.gluten.GlutenConfig + import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait @@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait { } testGluten("test customer column rules") { - withSQLConf(("spark.gluten.enabled", "false")) { + withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) { sql("create table my_parquet(id int) using parquet") sql("insert into my_parquet values (1)") sql("insert into my_parquet values (2)") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index b5481f4d88c4..471bdf1796cd 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -166,7 +166,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { } if (executedVanilla) { - spark.conf.set("spark.gluten.enabled", "false") + spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") val vanillaParquet = spark.sql(s""" |select $scanSchema from parquet.`$parquetDir` diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index 18a9f5c9fc8c..5e6c66265ad7 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.extension +import org.apache.gluten.GlutenConfig + import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait @@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait { } testGluten("test customer column rules") { - withSQLConf(("spark.gluten.enabled", "false")) { + withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) { sql("create table my_parquet(id int) using parquet") sql("insert into my_parquet values (1)") sql("insert into my_parquet values (2)") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 7d8a292042e8..f61f7ebba9b2 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -165,7 +165,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { } if (executedVanilla) { - spark.conf.set("spark.gluten.enabled", "false") + spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") val vanillaParquet = spark.sql(s""" |select $scanSchema from parquet.`$parquetDir` diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index 18a9f5c9fc8c..5e6c66265ad7 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.extension +import org.apache.gluten.GlutenConfig + import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait @@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait { } testGluten("test customer column rules") { - withSQLConf(("spark.gluten.enabled", "false")) { + withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) { sql("create table my_parquet(id int) using parquet") sql("insert into my_parquet values (1)") sql("insert into my_parquet values (2)") From 621a479de0496d65b4bd76a37f613f67ceaaabcb Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 23 May 2024 08:29:01 +0800 Subject: [PATCH 132/402] [GLUTEN-5771][VL] Add metrics for ColumnarArrowEvalPythonExec (#5772) This patch adds metric for ColumnarArrowEvalPythonExec, also a brief guide on how to use pandas UDF is added. --------- Co-authored-by: Hongze Zhang --- .../python/ColumnarArrowEvalPythonExec.scala | 42 ++++++++++++++++- .../{VeloxNativeUDF.md => VeloxUDF.md} | 45 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) rename backends-velox/src/main/scala/org/apache/{gluten/execution => spark/api}/python/ColumnarArrowEvalPythonExec.scala (90%) rename docs/developers/{VeloxNativeUDF.md => VeloxUDF.md} (78%) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala similarity index 90% rename from backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala rename to backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala index fd8dfc25b89d..d5639057dac8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/python/ColumnarArrowEvalPythonExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala @@ -28,6 +28,7 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} +import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.execution.python.{ArrowEvalPythonExec, BasePythonRunnerShim, EvalPythonExec, PythonUDFRunner} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructField, StructType} @@ -209,7 +210,13 @@ case class ColumnarArrowEvalPythonExec( extends EvalPythonExec with GlutenPlan { override def supportsColumnar: Boolean = true - // FIXME: incorrect metrics updater + + override lazy val metrics = Map( + "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), + "numOutputBatches" -> SQLMetrics.createMetric(sparkContext, "output_batches"), + "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"), + "processTime" -> SQLMetrics.createTimingMetric(sparkContext, "totaltime_arrow_udf") + ) override protected def evaluate( funcs: Seq[ChainedPythonFunctions], @@ -277,6 +284,10 @@ case class ColumnarArrowEvalPythonExec( } override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val numOutputRows = longMetric("numOutputRows") + val numOutputBatches = longMetric("numOutputBatches") + val numInputRows = longMetric("numInputRows") + val procTime = longMetric("processTime") val inputRDD = child.executeColumnar() inputRDD.mapPartitions { iter => @@ -318,12 +329,15 @@ case class ColumnarArrowEvalPythonExec( val contextAwareIterator = new ContextAwareIterator(context, iter) val inputCbCache = new ArrayBuffer[ColumnarBatch]() + var start_time: Long = 0 val inputBatchIter = contextAwareIterator.map { inputCb => + start_time = System.nanoTime() ColumnarBatches.ensureLoaded(ArrowBufferAllocators.contextInstance, inputCb) ColumnarBatches.retain(inputCb) // 0. cache input for later merge inputCbCache += inputCb + numInputRows += inputCb.numRows // We only need to pass the referred cols data to python worker for evaluation. var colsForEval = new ArrayBuffer[ColumnVector]() for (i <- originalOffsets) { @@ -341,11 +355,20 @@ case class ColumnarArrowEvalPythonExec( val joinedVectors = (0 until inputCb.numCols).toArray.map( i => inputCb.column(i)) ++ (0 until outputCb.numCols).toArray.map( i => outputCb.column(i)) + // Columns in outputCb has random 0 or 1 refCnt and will fail checks in ensureOffload, + // so we do a hard reset here. + (0 until joinedVectors.length).foreach( + i => { + adjustRefCnt(joinedVectors(i).asInstanceOf[ArrowWritableColumnVector], 1) + }) val numRows = inputCb.numRows + numOutputBatches += 1 + numOutputRows += numRows val batch = new ColumnarBatch(joinedVectors, numRows) val offloaded = ColumnarBatches.ensureOffloaded(ArrowBufferAllocators.contextInstance, batch) ColumnarBatches.release(outputCb) + procTime += (System.nanoTime() - start_time) / 1000000 offloaded } Iterators @@ -358,6 +381,23 @@ case class ColumnarArrowEvalPythonExec( } } + private def adjustRefCnt(vector: ArrowWritableColumnVector, to: Long): Unit = { + val from = vector.refCnt() + if (from == to) { + return + } + if (from > to) { + do { + vector.close() + } while (vector.refCnt() == to) + return + } + // from < to + do { + vector.retain() + } while (vector.refCnt() == to) + } + override protected def withNewChildInternal(newChild: SparkPlan): ColumnarArrowEvalPythonExec = copy(udfs, resultAttrs, newChild) } diff --git a/docs/developers/VeloxNativeUDF.md b/docs/developers/VeloxUDF.md similarity index 78% rename from docs/developers/VeloxNativeUDF.md rename to docs/developers/VeloxUDF.md index b951905931e5..b88c4de1515e 100644 --- a/docs/developers/VeloxNativeUDF.md +++ b/docs/developers/VeloxUDF.md @@ -179,3 +179,48 @@ The output from spark-shell will be like +------------------+----------------+ ``` +# Pandas UDFs (a.k.a. Vectorized UDFs) + +## Introduction + +Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF is defined using the pandas_udf() as a decorator or to wrap the function, and no additional configuration is required. +A Pandas UDF behaves as a regular PySpark function API in general. For more details, you can refer [doc](https://spark.apache.org/docs/latest/api/python/user_guide/sql/arrow_pandas.html). + +## Using Pandas UDFs in Gluten with Velox Backend + +Similar as in vanilla Spark, user needs to set up pyspark/arrow dependencies properly first. You may can refer following steps: + +``` +pip3 install pyspark==$SPARK_VERSION cython +pip3 install pandas pyarrow +``` + +Gluten provides a config to control enable `ColumnarArrowEvalPython` or not, with `true` as defalt. + +``` +spark.gluten.sql.columnar.arrowUdf +``` + +Then take following `PySpark` code for example: + +``` +from pyspark.sql.functions import pandas_udf, PandasUDFType +import pyspark.sql.functions as F +import os +@pandas_udf('long') +def pandas_plus_one(v): + return (v + 1) +df = spark.read.orc("path_to_file").select("quantity").withColumn("processed_quantity", pandas_plus_one("quantity")).select("quantity") +``` + +The expected physical plan will be: + +``` +== Physical Plan == +VeloxColumnarToRowExec ++- ^(2) ProjectExecTransformer [pythonUDF0#45L AS processed_quantity#41L] + +- ^(2) InputIteratorTransformer[quantity#2L, pythonUDF0#45L] + +- ^(2) InputAdapter + +- ^(2) ColumnarArrowEvalPython [pandas_plus_one(quantity#2L)#40L], [pythonUDF0#45L], 200 + +- ^(1) NativeFileScan orc [quantity#2L] Batched: true, DataFilters: [], Format: ORC, Location: InMemoryFileIndex(1 paths)[file:/***], PartitionFilters: [], PushedFilters: [], ReadSchema: struct +``` From 7c777be25fcd549dd53653986c6d40bd6cdcb965 Mon Sep 17 00:00:00 2001 From: Yuan Date: Thu, 23 May 2024 08:40:04 +0800 Subject: [PATCH 133/402] [VL] Upgrade cmake version to 3.28.3 in CI image (#5842) --- .github/workflows/velox_docker.yml | 2 +- dev/ci-velox-buildstatic.sh | 2 +- dev/vcpkg/init.sh | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 88c6c2a241b4..d7644b5d0be5 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -49,7 +49,7 @@ concurrency: jobs: build-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_03_17 # centos7 with dependencies installed + container: apache/gluten:gluten-vcpkg-builder_2024_05_22 # centos7 with dependencies installed steps: - uses: actions/checkout@v2 - name: Generate cache key diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic.sh index a9b9d2c3fcc7..208490d1c2eb 100755 --- a/dev/ci-velox-buildstatic.sh +++ b/dev/ci-velox-buildstatic.sh @@ -2,7 +2,7 @@ yum install sudo patch java-1.8.0-openjdk-devel -y cd $GITHUB_WORKSPACE/ep/build-velox/src ./get_velox.sh source /opt/rh/devtoolset-9/enable -source $GITHUB_WORKSPACE//dev/vcpkg/env.sh +source /opt/gluten/dev/vcpkg/env.sh cd $GITHUB_WORKSPACE/ sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt export NUM_THREADS=4 diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh index 141543af44c0..e69aec94ab1f 100755 --- a/dev/vcpkg/init.sh +++ b/dev/vcpkg/init.sh @@ -16,6 +16,9 @@ if [ ! -d "$VCPKG_ROOT" ] || [ -z "$(ls "$VCPKG_ROOT")" ]; then fi [ -f "$VCPKG" ] || "$VCPKG_ROOT/bootstrap-vcpkg.sh" -disableMetrics +sed -i "s/3.27.1/3.28.3/g" $VCPKG_ROOT/scripts/vcpkgTools.xml +sed -i "s/192374a68e2971f04974a194645726196d9b8ee7abd650d1e6f65f7aa2ccc9b186c3edb473bb4958c764532edcdd42f4182ee1fcb86b17d78b0bcd6305ce3df1/bd311ca835ef0914952f21d70d1753564d58de2ede02e80ede96e78cd2f40b4189e006007643ebb37792e13edd97eb4a33810bc8aca1eab6dd428eaffe1d2e38/g" $VCPKG_ROOT/scripts/vcpkgTools.xml + $VCPKG install --no-print-usage \ --triplet="${VCPKG_TRIPLET}" --host-triplet="${VCPKG_TRIPLET}" From 90961bc907955409e1f3b7c09af00aa3bf7abf16 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 23 May 2024 08:45:33 +0800 Subject: [PATCH 134/402] [VL] RAS: Optimize offload rule code to gain better compatibility with rewrite rules (#5836) --- .../enumerated/EnumeratedTransform.scala | 59 ++++--- .../enumerated/PushFilterToScan.scala | 6 +- .../columnar/enumerated/RasOffload.scala | 147 ++++++++++++------ .../enumerated/RasOffloadFilter.scala | 5 +- ...te.scala => RasOffloadHashAggregate.scala} | 7 +- .../columnar/enumerated/RemoveFilter.scala | 2 +- .../columnar/transition/Transition.scala | 2 +- .../org/apache/gluten/ras/path/Pattern.scala | 22 ++- .../apache/gluten/ras/path/WizardSuite.scala | 21 ++- .../apache/gluten/ras/rule/PatternSuite.scala | 37 ++--- 10 files changed, 206 insertions(+), 102 deletions(-) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/{RasOffloadAggregate.scala => RasOffloadHashAggregate.scala} (83%) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index 50f0dce13dc7..c41c1ca2caa0 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -16,21 +16,29 @@ */ package org.apache.gluten.extension.columnar.enumerated -import org.apache.gluten.extension.columnar.{OffloadExchange, OffloadJoin, OffloadOthers, OffloadSingleNode} +import org.apache.gluten.extension.columnar.{OffloadExchange, OffloadJoin, OffloadOthers} import org.apache.gluten.extension.columnar.transition.ConventionReq import org.apache.gluten.planner.GlutenOptimization import org.apache.gluten.planner.property.Conv import org.apache.gluten.ras.property.PropertySet +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.LogLevelUtil import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.aggregate.{ObjectHashAggregateExec, SortAggregateExec} +import org.apache.spark.sql.execution.datasources.WriteFilesExec +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase +import org.apache.spark.sql.execution.exchange.Exchange +import org.apache.spark.sql.execution.joins.BaseJoinExec +import org.apache.spark.sql.execution.python.EvalPythonExec +import org.apache.spark.sql.execution.window.WindowExec +import org.apache.spark.sql.hive.HiveTableScanExecTransformer case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) extends Rule[SparkPlan] with LogLevelUtil { - import EnumeratedTransform._ private val rules = List( new PushFilterToScan(RasOffload.validator), @@ -40,11 +48,35 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) // TODO: Should obey ReplaceSingleNode#applyScanNotTransformable to select // (vanilla) scan with cheaper sub-query plan through cost model. private val offloadRules = List( - new AsRasOffload(OffloadOthers()), - new AsRasOffload(OffloadExchange()), - new AsRasOffload(OffloadJoin()), - RasOffloadAggregate, - RasOffloadFilter + RasOffload.from[Exchange](OffloadExchange()).toRule, + RasOffload.from[BaseJoinExec](OffloadJoin()).toRule, + RasOffloadHashAggregate.toRule, + RasOffloadFilter.toRule, + RasOffload.from[DataSourceV2ScanExecBase](OffloadOthers()).toRule, + RasOffload.from[DataSourceScanExec](OffloadOthers()).toRule, + RasOffload + .from( + (node: SparkPlan) => HiveTableScanExecTransformer.isHiveTableScan(node), + OffloadOthers()) + .toRule, + RasOffload.from[CoalesceExec](OffloadOthers()).toRule, + RasOffload.from[ProjectExec](OffloadOthers()).toRule, + RasOffload.from[SortAggregateExec](OffloadOthers()).toRule, + RasOffload.from[ObjectHashAggregateExec](OffloadOthers()).toRule, + RasOffload.from[UnionExec](OffloadOthers()).toRule, + RasOffload.from[ExpandExec](OffloadOthers()).toRule, + RasOffload.from[WriteFilesExec](OffloadOthers()).toRule, + RasOffload.from[SortExec](OffloadOthers()).toRule, + RasOffload.from[TakeOrderedAndProjectExec](OffloadOthers()).toRule, + RasOffload.from[WindowExec](OffloadOthers()).toRule, + RasOffload + .from( + (node: SparkPlan) => SparkShimLoader.getSparkShims.isWindowGroupLimitExec(node), + OffloadOthers()) + .toRule, + RasOffload.from[LimitExec](OffloadOthers()).toRule, + RasOffload.from[GenerateExec](OffloadOthers()).toRule, + RasOffload.from[EvalPythonExec](OffloadOthers()).toRule ) private val optimization = GlutenOptimization(rules ++ offloadRules) @@ -67,13 +99,4 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) } } -object EnumeratedTransform { - - /** Accepts a [[OffloadSingleNode]] rule to convert it into a RAS offload rule. */ - private class AsRasOffload(delegate: OffloadSingleNode) extends RasOffload { - override protected def offload(node: SparkPlan): SparkPlan = { - val out = delegate.offload(node) - out - } - } -} +object EnumeratedTransform {} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala index 388668287091..611d6db0bd48 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala @@ -50,16 +50,16 @@ class PushFilterToScan(validator: Validator) extends RasRule[SparkPlan] { override def shape(): Shape[SparkPlan] = anyOf( pattern( - node[SparkPlan]( + branch[SparkPlan]( clazz(classOf[FilterExec]), leaf( or(clazz(classOf[FileSourceScanExec]), clazz(classOf[BatchScanExec])) ) ).build()), pattern( - node[SparkPlan]( + branch[SparkPlan]( clazz(classOf[FilterExec]), - node( + branch( clazz(classOf[ColumnarToRowTransition]), leaf( or(clazz(classOf[FileSourceScanExec]), clazz(classOf[BatchScanExec])) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala index 5cabfa88e700..6af89dc057aa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala @@ -17,61 +17,39 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.extension.columnar.OffloadSingleNode import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode import org.apache.gluten.extension.columnar.validator.{Validator, Validators} -import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} +import org.apache.gluten.ras.path.Pattern +import org.apache.gluten.ras.path.Pattern.node +import org.apache.gluten.ras.rule.{RasRule, Shape} +import org.apache.gluten.ras.rule.Shapes.pattern import org.apache.spark.sql.execution.SparkPlan -trait RasOffload extends RasRule[SparkPlan] { - import RasOffload._ +import scala.reflect.{classTag, ClassTag} - final override def shift(node: SparkPlan): Iterable[SparkPlan] = { - // 0. If the node is already offloaded, return fast. - if (node.isInstanceOf[GlutenPlan]) { - return List.empty - } +trait RasOffload { + def offload(plan: SparkPlan): SparkPlan + def typeIdentifier(): RasOffload.TypeIdentifier +} - // 1. Rewrite the node to form that native library supports. - val rewritten = rewrites.foldLeft(node) { - case (node, rewrite) => - node.transformUp { - case p => - val out = rewrite.rewrite(p) - out - } - } +object RasOffload { + trait TypeIdentifier { + def isInstance(node: SparkPlan): Boolean + } - // 2. Walk the rewritten tree. - val offloaded = rewritten.transformUp { - case from => - // 3. Validate current node. If passed, offload it. - validator.validate(from) match { - case Validator.Passed => - offload(from) match { - case t: GlutenPlan if !t.doValidate().isValid => - // 4. If native validation fails on the offloaded node, return the - // original one. - from - case other => - other - } - case Validator.Failed(reason) => - from - } + object TypeIdentifier { + def of[T <: SparkPlan: ClassTag]: TypeIdentifier = { + val nodeClass: Class[SparkPlan] = + classTag[T].runtimeClass.asInstanceOf[Class[SparkPlan]] + new TypeIdentifier { + override def isInstance(node: SparkPlan): Boolean = nodeClass.isInstance(node) + } } - - // 5. Return the final tree. - List(offloaded) } - protected def offload(node: SparkPlan): SparkPlan - - final override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) -} - -object RasOffload { - val validator = Validators + val validator: Validator = Validators .builder() .fallbackByHint() .fallbackIfScanOnly() @@ -82,4 +60,85 @@ object RasOffload { .build() private val rewrites = RewriteSingleNode.allRules() + + def from[T <: SparkPlan: ClassTag](base: OffloadSingleNode): RasOffload = { + new RasOffload { + override def offload(plan: SparkPlan): SparkPlan = base.offload(plan) + override def typeIdentifier(): TypeIdentifier = TypeIdentifier.of[T] + } + } + + def from(identifier: TypeIdentifier, base: OffloadSingleNode): RasOffload = { + new RasOffload { + override def offload(plan: SparkPlan): SparkPlan = base.offload(plan) + override def typeIdentifier(): TypeIdentifier = identifier + } + } + + implicit class RasOffloadOps(base: RasOffload) { + def toRule: RasRule[SparkPlan] = { + new RuleImpl(base) + } + } + + private class RuleImpl(base: RasOffload) extends RasRule[SparkPlan] { + private val typeIdentifier: TypeIdentifier = base.typeIdentifier() + + final override def shift(node: SparkPlan): Iterable[SparkPlan] = { + // 0. If the node is already offloaded, fail fast. + assert(typeIdentifier.isInstance(node)) + + // 1. Rewrite the node to form that native library supports. + val rewritten = rewrites.foldLeft(node) { + case (node, rewrite) => + node.transformUp { + case p => + val out = rewrite.rewrite(p) + out + } + } + + // 2. Walk the rewritten tree. + val offloaded = rewritten.transformUp { + case from if typeIdentifier.isInstance(from) => + // 3. Validate current node. If passed, offload it. + validator.validate(from) match { + case Validator.Passed => + val offloaded = base.offload(from) + offloaded match { + case t: GlutenPlan if !t.doValidate().isValid => + // 4. If native validation fails on the offloaded node, return the + // original one. + from + case other => + other + } + case Validator.Failed(reason) => + from + } + } + + // 5. If rewritten plan is not offload-able, discard it. + if (offloaded.fastEquals(rewritten)) { + return List.empty + } + + // 6. Otherwise, return the final tree. + List(offloaded) + } + + override def shape(): Shape[SparkPlan] = { + pattern(node[SparkPlan](new Pattern.Matcher[SparkPlan] { + override def apply(plan: SparkPlan): Boolean = { + if (plan.isInstanceOf[GlutenPlan]) { + return false + } + if (typeIdentifier.isInstance(plan)) { + return true + } + false + } + }).build()) + } + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala index 030d05d478f3..54ab9158be6a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadFilter.scala @@ -21,7 +21,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.spark.sql.execution.{FilterExec, SparkPlan} object RasOffloadFilter extends RasOffload { - override protected def offload(node: SparkPlan): SparkPlan = node match { + override def offload(node: SparkPlan): SparkPlan = node match { case FilterExec(condition, child) => val out = BackendsApiManager.getSparkPlanExecApiInstance .genFilterExecTransformer(condition, child) @@ -29,4 +29,7 @@ object RasOffloadFilter extends RasOffload { case other => other } + + override def typeIdentifier(): RasOffload.TypeIdentifier = + RasOffload.TypeIdentifier.of[FilterExec] } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadAggregate.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadHashAggregate.scala similarity index 83% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadAggregate.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadHashAggregate.scala index e48545ae9499..6c125478bd01 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadAggregate.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadHashAggregate.scala @@ -21,11 +21,14 @@ import org.apache.gluten.execution.HashAggregateExecBaseTransformer import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.HashAggregateExec -object RasOffloadAggregate extends RasOffload { - override protected def offload(node: SparkPlan): SparkPlan = node match { +object RasOffloadHashAggregate extends RasOffload { + override def offload(node: SparkPlan): SparkPlan = node match { case agg: HashAggregateExec => val out = HashAggregateExecBaseTransformer.from(agg)() out case other => other } + + override def typeIdentifier(): RasOffload.TypeIdentifier = + RasOffload.TypeIdentifier.of[HashAggregateExec] } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index c9f4b27bf203..46b3b7f9e088 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -42,7 +42,7 @@ object RemoveFilter extends RasRule[SparkPlan] { override def shape(): Shape[SparkPlan] = pattern( - node[SparkPlan]( + branch[SparkPlan]( clazz(classOf[FilterExecTransformerBase]), leaf(clazz(classOf[BasicScanExecTransformer])) ).build()) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala index 73a126f8df27..3fd2839b5a0b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/transition/Transition.scala @@ -31,7 +31,7 @@ import scala.collection.mutable trait Transition { final def apply(plan: SparkPlan): SparkPlan = { val out = apply0(plan) - if (out.fastEquals(plan)) { + if (out eq plan) { assert( this == Transition.empty, "TransitionDef.empty / Transition.empty should be used when defining an empty transition.") diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala index f20d05c7c00c..e60a94717654 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala @@ -52,7 +52,7 @@ object Pattern { def children(count: Int): Seq[Node[T]] } - private case class Any[T <: AnyRef]() extends Node[Null] { + private case class Any private () extends Node[Null] { override def skip(): Boolean = false override def abort(node: CanonicalNode[Null]): Boolean = false override def matches(node: CanonicalNode[Null]): Boolean = true @@ -60,12 +60,12 @@ object Pattern { } private object Any { - val INSTANCE: Any[Null] = Any[Null]() + val INSTANCE: Node[Null] = new Any() // Enclose default constructor. - private def apply[T <: AnyRef](): Any[T] = new Any() + private def apply(): Node[Null] = throw new UnsupportedOperationException() } - private case class Ignore[T <: AnyRef]() extends Node[Null] { + private case class Ignore private () extends Node[Null] { override def skip(): Boolean = true override def abort(node: CanonicalNode[Null]): Boolean = false override def matches(node: CanonicalNode[Null]): Boolean = @@ -74,10 +74,17 @@ object Pattern { } private object Ignore { - val INSTANCE: Ignore[Null] = Ignore[Null]() + val INSTANCE: Node[Null] = new Ignore() // Enclose default constructor. - private def apply[T <: AnyRef](): Ignore[T] = new Ignore() + private def apply(): Node[Null] = throw new UnsupportedOperationException() + } + + private case class Single[T <: AnyRef](matcher: Matcher[T]) extends Node[T] { + override def skip(): Boolean = false + override def abort(node: CanonicalNode[T]): Boolean = false + override def matches(node: CanonicalNode[T]): Boolean = matcher(node.self()) + override def children(count: Int): Seq[Node[T]] = (0 until count).map(_ => ignore[T]) } private case class Branch[T <: AnyRef](matcher: Matcher[T], children: Seq[Node[T]]) @@ -93,7 +100,8 @@ object Pattern { def any[T <: AnyRef]: Node[T] = Any.INSTANCE.asInstanceOf[Node[T]] def ignore[T <: AnyRef]: Node[T] = Ignore.INSTANCE.asInstanceOf[Node[T]] - def node[T <: AnyRef](matcher: Matcher[T], children: Node[T]*): Node[T] = + def node[T <: AnyRef](matcher: Matcher[T]): Node[T] = Single(matcher) + def branch[T <: AnyRef](matcher: Matcher[T], children: Node[T]*): Node[T] = Branch(matcher, children.toSeq) def leaf[T <: AnyRef](matcher: Matcher[T]): Node[T] = Branch(matcher, List.empty) diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/path/WizardSuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/path/WizardSuite.scala index 523a22689ea6..59cc44600b5e 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/path/WizardSuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/path/WizardSuite.scala @@ -198,18 +198,18 @@ class WizardSuite extends AnyFunSuite { findWithPatterns( List( Pattern - .node[TestNode]( + .branch[TestNode]( _ => true, - Pattern.node(_ => true, Pattern.ignore), - Pattern.node(_ => true, Pattern.ignore)) + Pattern.branch(_ => true, Pattern.ignore), + Pattern.branch(_ => true, Pattern.ignore)) .build())) == List(Binary(n1, Unary(n2, Group(3)), Unary(n3, Group(4))))) // Pattern pruning should emit all results val pattern1 = Pattern - .node[TestNode](_ => true, Pattern.node(_ => true, Pattern.ignore), Pattern.ignore) + .branch[TestNode](_ => true, Pattern.branch(_ => true, Pattern.ignore), Pattern.ignore) .build() val pattern2 = Pattern - .node[TestNode](_ => true, Pattern.ignore, Pattern.node(_ => true, Pattern.ignore)) + .branch[TestNode](_ => true, Pattern.ignore, Pattern.branch(_ => true, Pattern.ignore)) .build() assert( @@ -219,10 +219,10 @@ class WizardSuite extends AnyFunSuite { // Distinguish between ignore and any val pattern3 = Pattern - .node[TestNode](_ => true, Pattern.node(_ => true, Pattern.any), Pattern.ignore) + .branch[TestNode](_ => true, Pattern.branch(_ => true, Pattern.any), Pattern.ignore) .build() val pattern4 = Pattern - .node[TestNode](_ => true, Pattern.ignore, Pattern.node(_ => true, Pattern.any)) + .branch[TestNode](_ => true, Pattern.ignore, Pattern.branch(_ => true, Pattern.any)) .build() assert( @@ -231,6 +231,13 @@ class WizardSuite extends AnyFunSuite { Binary(n1, Group(1), Unary(n3, Leaf(n6, 1))), Binary(n1, Unary(n2, Leaf(n4, 1)), Group(2)))) + // Single + val pattern5 = Pattern.node[TestNode](_ => true).build() + assert(findWithPatterns(List(pattern5)) == List(Binary(n1, Group(1), Group(2)))) + val pattern6 = Pattern.node[TestNode](_.isInstanceOf[Binary]).build() + assert(findWithPatterns(List(pattern6)) == List(Binary(n1, Group(1), Group(2)))) + val pattern7 = Pattern.node[TestNode](_.isInstanceOf[Leaf]).build() + assert(findWithPatterns(List(pattern7)).isEmpty) } test("Prune by mask") { diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala index 2a86f164d9de..64b66bbaffae 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala @@ -72,12 +72,12 @@ class PatternSuite extends AnyFunSuite { val path = MockRasPath.mock(ras, Unary("n1", Leaf("n2", 1))) assert(path.height() == 2) - val pattern1 = Pattern.node[TestNode](n => n.isInstanceOf[Unary], Pattern.ignore).build() + val pattern1 = Pattern.branch[TestNode](n => n.isInstanceOf[Unary], Pattern.ignore).build() assert(pattern1.matches(path, 1)) assert(pattern1.matches(path, 2)) val pattern2 = - Pattern.node[TestNode](n => n.asInstanceOf[Unary].name == "foo", Pattern.ignore).build() + Pattern.branch[TestNode](n => n.asInstanceOf[Unary].name == "foo", Pattern.ignore).build() assert(!pattern2.matches(path, 1)) assert(!pattern2.matches(path, 2)) } @@ -98,11 +98,11 @@ class PatternSuite extends AnyFunSuite { assert(path.height() == 4) val pattern = Pattern - .node[TestNode]( + .branch[TestNode]( n => n.isInstanceOf[Binary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], Pattern.ignore ) @@ -131,11 +131,11 @@ class PatternSuite extends AnyFunSuite { assert(path.height() == 4) val pattern1 = Pattern - .node[TestNode]( + .branch[TestNode]( n => n.isInstanceOf[Binary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], Pattern.leaf( _.asInstanceOf[Leaf].name == "foo" @@ -152,13 +152,13 @@ class PatternSuite extends AnyFunSuite { assert(!pattern1.matches(path, 4)) val pattern2 = Pattern - .node[TestNode]( + .branch[TestNode]( n => n.isInstanceOf[Binary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], - Pattern.node( + Pattern.branch( n => n.isInstanceOf[Unary], Pattern.ignore ) @@ -188,9 +188,9 @@ class PatternSuite extends AnyFunSuite { assert(path.height() == 2) val pattern1 = Pattern - .node[TestNode]( + .branch[TestNode]( Pattern.Matchers.clazz(classOf[Unary]), - Pattern.node(Pattern.Matchers.clazz(classOf[Leaf]))) + Pattern.branch(Pattern.Matchers.clazz(classOf[Leaf]))) .build() assert(pattern1.matches(path, 1)) assert(pattern1.matches(path, 2)) @@ -202,19 +202,20 @@ class PatternSuite extends AnyFunSuite { assert(!pattern2.matches(path, 2)) val pattern3 = Pattern - .node[TestNode]( + .branch[TestNode]( Pattern.Matchers .or(Pattern.Matchers.clazz(classOf[Unary]), Pattern.Matchers.clazz(classOf[Leaf])), - Pattern.node(Pattern.Matchers.clazz(classOf[Leaf]))) + Pattern.branch(Pattern.Matchers.clazz(classOf[Leaf])) + ) .build() assert(pattern3.matches(path, 1)) assert(pattern3.matches(path, 2)) val pattern4 = Pattern - .node[TestNode]( + .branch[TestNode]( Pattern.Matchers .or(Pattern.Matchers.clazz(classOf[Unary]), Pattern.Matchers.clazz(classOf[Leaf])), - Pattern.node(Pattern.Matchers + Pattern.branch(Pattern.Matchers .or(Pattern.Matchers.clazz(classOf[Unary]), Pattern.Matchers.clazz(classOf[Unary]))) ) .build() From fbb4ec588b0d1ef88d179632de598ac77091367b Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Thu, 23 May 2024 12:28:43 +0800 Subject: [PATCH 135/402] [VL] Enable local sort-based shuffle (#5811) --- .../backendsapi/clickhouse/CHMetricsApi.scala | 3 ++- .../clickhouse/CHSparkPlanExecApi.scala | 6 +++-- .../backendsapi/velox/VeloxMetricsApi.scala | 18 ++++++++----- .../velox/VeloxSparkPlanExecApi.scala | 22 ++++++++++------ cpp/core/jni/JniWrapper.cc | 3 --- cpp/core/shuffle/LocalPartitionWriter.cc | 26 +++++++++++++++---- cpp/core/shuffle/LocalPartitionWriter.h | 2 ++ cpp/core/shuffle/Payload.cc | 4 +++ cpp/core/shuffle/Payload.h | 2 +- cpp/core/shuffle/ShuffleReader.cc | 4 --- cpp/core/shuffle/ShuffleReader.h | 1 - cpp/core/shuffle/Spill.cc | 3 +++ cpp/velox/shuffle/VeloxShuffleReader.cc | 26 +++++-------------- cpp/velox/shuffle/VeloxShuffleReader.h | 10 +++---- .../shuffle/VeloxSortBasedShuffleWriter.cc | 3 --- cpp/velox/tests/VeloxShuffleWriterTest.cc | 2 ++ .../utils/tests/VeloxShuffleWriterTestBase.h | 3 +-- docs/Configuration.md | 3 ++- .../gluten/backendsapi/MetricsApi.scala | 4 ++- .../gluten/backendsapi/SparkPlanExecApi.scala | 8 ++++-- .../shuffle/ColumnarShuffleDependency.scala | 3 ++- .../shuffle/GlutenShuffleWriterWrapper.scala | 15 ++++++++--- .../ColumnarShuffleExchangeExec.scala | 19 ++++++++++---- .../vectorized/ShuffleReaderMetrics.java | 9 ------- .../vectorized/ShuffleWriterJniWrapper.java | 5 ++-- .../vectorized/ColumnarBatchSerializer.scala | 26 +++++++++++-------- .../spark/shuffle/ColumnarShuffleWriter.scala | 12 ++++++--- .../spark/shuffle/utils/ShuffleUtil.scala | 3 ++- .../spark/sql/execution/utils/ExecUtil.scala | 6 +++-- .../gluten/uniffle/UniffleShuffleManager.java | 5 +++- .../VeloxUniffleColumnarShuffleWriter.java | 7 +++-- .../org/apache/gluten/GlutenConfig.scala | 10 +++++++ 32 files changed, 168 insertions(+), 105 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala index d41f370cfb21..30f682f0fb2e 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala @@ -208,7 +208,8 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) override def genColumnarShuffleExchangeMetrics( - sparkContext: SparkContext): Map[String, SQLMetric] = + sparkContext: SparkContext, + isSort: Boolean): Map[String, SQLMetric] = Map( "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"), "bytesSpilled" -> SQLMetrics.createSizeMetric(sparkContext, "shuffle bytes spilled"), diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 45f90719fa41..6a154cd945e0 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -407,7 +407,8 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { newPartitioning: Partitioning, serializer: Serializer, writeMetrics: Map[String, SQLMetric], - metrics: Map[String, SQLMetric] + metrics: Map[String, SQLMetric], + isSort: Boolean ): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = { CHExecUtil.genShuffleDependency( rdd, @@ -438,7 +439,8 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { */ override def createColumnarBatchSerializer( schema: StructType, - metrics: Map[String, SQLMetric]): Serializer = { + metrics: Map[String, SQLMetric], + isSort: Boolean): Serializer = { val readBatchNumRows = metrics("avgReadBatchNumRows") val numOutputRows = metrics("numOutputRows") val dataSize = metrics("dataSize") diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala index 4afb59a43fda..7be639d4caf5 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala @@ -259,18 +259,15 @@ class VeloxMetricsApi extends MetricsApi with Logging { Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) override def genColumnarShuffleExchangeMetrics( - sparkContext: SparkContext): Map[String, SQLMetric] = - Map( + sparkContext: SparkContext, + isSort: Boolean): Map[String, SQLMetric] = { + val baseMetrics = Map( "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"), "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions"), "bytesSpilled" -> SQLMetrics.createSizeMetric(sparkContext, "shuffle bytes spilled"), "splitBufferSize" -> SQLMetrics.createSizeMetric(sparkContext, "split buffer size total"), "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to split"), "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to spill"), - "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to compress"), - "prepareTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to prepare"), - "decompressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime decompress"), - "ipcTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime ipc"), "deserializeTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime deserialize"), "avgReadBatchNumRows" -> SQLMetrics .createAverageMetric(sparkContext, "avg read batch num rows"), @@ -280,6 +277,15 @@ class VeloxMetricsApi extends MetricsApi with Logging { "inputBatches" -> SQLMetrics .createMetric(sparkContext, "number of input batches") ) + if (isSort) { + baseMetrics + } else { + baseMetrics ++ Map( + "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to compress"), + "decompressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime decompress") + ) + } + } override def genWindowTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = Map( diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 3221165827ca..cfa135046012 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -457,7 +457,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { newPartitioning: Partitioning, serializer: Serializer, writeMetrics: Map[String, SQLMetric], - metrics: Map[String, SQLMetric]): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = { + metrics: Map[String, SQLMetric], + isSort: Boolean): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = { // scalastyle:on argcount ExecUtil.genShuffleDependency( rdd, @@ -465,7 +466,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { newPartitioning, serializer, writeMetrics, - metrics) + metrics, + isSort) } // scalastyle:on argcount @@ -510,12 +512,16 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { */ override def createColumnarBatchSerializer( schema: StructType, - metrics: Map[String, SQLMetric]): Serializer = { - val readBatchNumRows = metrics("avgReadBatchNumRows") + metrics: Map[String, SQLMetric], + isSort: Boolean): Serializer = { val numOutputRows = metrics("numOutputRows") - val decompressTime = metrics("decompressTime") - val ipcTime = metrics("ipcTime") val deserializeTime = metrics("deserializeTime") + val readBatchNumRows = metrics("avgReadBatchNumRows") + val decompressTime: Option[SQLMetric] = if (!isSort) { + Some(metrics("decompressTime")) + } else { + None + } if (GlutenConfig.getConf.isUseCelebornShuffleManager) { val clazz = ClassUtils.getClass("org.apache.spark.shuffle.CelebornColumnarBatchSerializer") val constructor = @@ -526,9 +532,9 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { schema, readBatchNumRows, numOutputRows, + deserializeTime, decompressTime, - ipcTime, - deserializeTime) + isSort) } } diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index b1edfbd019b2..a04ba73a18af 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -70,7 +70,6 @@ static jmethodID nativeColumnarToRowInfoConstructor; static jclass shuffleReaderMetricsClass; static jmethodID shuffleReaderMetricsSetDecompressTime; -static jmethodID shuffleReaderMetricsSetIpcTime; static jmethodID shuffleReaderMetricsSetDeserializeTime; static jclass block_stripes_class; @@ -278,7 +277,6 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { createGlobalClassReferenceOrError(env, "Lorg/apache/gluten/vectorized/ShuffleReaderMetrics;"); shuffleReaderMetricsSetDecompressTime = getMethodIdOrError(env, shuffleReaderMetricsClass, "setDecompressTime", "(J)V"); - shuffleReaderMetricsSetIpcTime = getMethodIdOrError(env, shuffleReaderMetricsClass, "setIpcTime", "(J)V"); shuffleReaderMetricsSetDeserializeTime = getMethodIdOrError(env, shuffleReaderMetricsClass, "setDeserializeTime", "(J)V"); @@ -1108,7 +1106,6 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper auto reader = ctx->objectStore()->retrieve(shuffleReaderHandle); env->CallVoidMethod(metrics, shuffleReaderMetricsSetDecompressTime, reader->getDecompressTime()); - env->CallVoidMethod(metrics, shuffleReaderMetricsSetIpcTime, reader->getIpcTime()); env->CallVoidMethod(metrics, shuffleReaderMetricsSetDeserializeTime, reader->getDeserializeTime()); checkException(env); diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc b/cpp/core/shuffle/LocalPartitionWriter.cc index 2fa0b954fa5f..6c4a0af39a22 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.cc +++ b/cpp/core/shuffle/LocalPartitionWriter.cc @@ -45,10 +45,9 @@ class LocalPartitionWriter::LocalSpiller { arrow::Status spill(uint32_t partitionId, std::unique_ptr payload) { // Check spill Type. - if (payload->type() != Payload::kUncompressed) { - return arrow::Status::Invalid( - "Cannot spill payload of type: " + payload->toString() + ", must be Payload::kUncompressed."); - } + ARROW_RETURN_IF( + payload->type() != Payload::kUncompressed && payload->type() != Payload::kRaw, + arrow::Status::Invalid("Cannot spill payload of type: " + payload->toString())); if (!opened_) { opened_ = true; @@ -63,6 +62,10 @@ class LocalPartitionWriter::LocalSpiller { ARROW_ASSIGN_OR_RAISE(auto end, os_->Tell()); DLOG(INFO) << "LocalSpiller: Spilled partition " << partitionId << " file start: " << start << ", file end: " << end << ", file: " << spillFile_; + if (payload->type() == Payload::kRaw) { + diskSpill_->insertPayload(partitionId, Payload::kRaw, 0, nullptr, end - start, pool_, nullptr); + return arrow::Status::OK(); + } auto payloadType = codec_ != nullptr && payload->numRows() >= compressionThreshold_ ? Payload::kToBeCompressed : Payload::kUncompressed; @@ -542,7 +545,20 @@ arrow::Status LocalPartitionWriter::evict( } arrow::Status LocalPartitionWriter::evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) { - return arrow::Status::NotImplemented("Invalid code path for local shuffle writer: sort based is not supported."); + rawPartitionLengths_[partitionId] += rawSize; + + if (partitionId <= lastEvictPid_) { + RETURN_NOT_OK(finishSpill()); + } + lastEvictPid_ = partitionId; + + RETURN_NOT_OK(requestSpill()); + auto buffer = std::make_shared(reinterpret_cast(data), length); + ARROW_ASSIGN_OR_RAISE( + auto payload, BlockPayload::fromBuffers(Payload::kRaw, 0, {std::move(buffer)}, nullptr, nullptr, nullptr)); + RETURN_NOT_OK(spiller_->spill(partitionId, std::move(payload))); + + return arrow::Status::OK(); } arrow::Status LocalPartitionWriter::reclaimFixedSize(int64_t size, int64_t* actual) { diff --git a/cpp/core/shuffle/LocalPartitionWriter.h b/cpp/core/shuffle/LocalPartitionWriter.h index c2bfacd4b63e..71467cff11e2 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.h +++ b/cpp/core/shuffle/LocalPartitionWriter.h @@ -112,5 +112,7 @@ class LocalPartitionWriter : public PartitionWriter { int64_t totalBytesWritten_{0}; std::vector partitionLengths_; std::vector rawPartitionLengths_; + + uint32_t lastEvictPid_{0}; }; } // namespace gluten diff --git a/cpp/core/shuffle/Payload.cc b/cpp/core/shuffle/Payload.cc index cfcc740c0b5d..626ed0cf0c18 100644 --- a/cpp/core/shuffle/Payload.cc +++ b/cpp/core/shuffle/Payload.cc @@ -266,6 +266,10 @@ arrow::Status BlockPayload::serialize(arrow::io::OutputStream* outputStream) { RETURN_NOT_OK(outputStream->Write(&numRows_, sizeof(uint32_t))); RETURN_NOT_OK(outputStream->Write(std::move(buffers_[0]))); } break; + case Type::kRaw: { + ScopedTimer timer(&writeTime_); + RETURN_NOT_OK(outputStream->Write(std::move(buffers_[0]))); + } break; } buffers_.clear(); return arrow::Status::OK(); diff --git a/cpp/core/shuffle/Payload.h b/cpp/core/shuffle/Payload.h index fac1022d56c2..508660b772b3 100644 --- a/cpp/core/shuffle/Payload.h +++ b/cpp/core/shuffle/Payload.h @@ -28,7 +28,7 @@ namespace gluten { class Payload { public: - enum Type : uint8_t { kCompressed = 1, kUncompressed = 2, kToBeCompressed = 3 }; + enum Type : uint8_t { kCompressed = 1, kUncompressed = 2, kToBeCompressed = 3, kRaw = 4 }; Payload(Type type, uint32_t numRows, const std::vector* isValidityBuffer); diff --git a/cpp/core/shuffle/ShuffleReader.cc b/cpp/core/shuffle/ShuffleReader.cc index faa81b52206a..5605661df4a4 100644 --- a/cpp/core/shuffle/ShuffleReader.cc +++ b/cpp/core/shuffle/ShuffleReader.cc @@ -44,10 +44,6 @@ int64_t ShuffleReader::getDecompressTime() const { return factory_->getDecompressTime(); } -int64_t ShuffleReader::getIpcTime() const { - return ipcTime_; -} - ShuffleWriterType ShuffleReader::getShuffleWriterType() const { return factory_->getShuffleWriterType(); } diff --git a/cpp/core/shuffle/ShuffleReader.h b/cpp/core/shuffle/ShuffleReader.h index 5cef14768450..0f985c7da939 100644 --- a/cpp/core/shuffle/ShuffleReader.h +++ b/cpp/core/shuffle/ShuffleReader.h @@ -67,7 +67,6 @@ class ShuffleReader { protected: arrow::MemoryPool* pool_; int64_t decompressTime_ = 0; - int64_t ipcTime_ = 0; int64_t deserializeTime_ = 0; ShuffleWriterType shuffleWriterType_; diff --git a/cpp/core/shuffle/Spill.cc b/cpp/core/shuffle/Spill.cc index a1abf36778d1..51e07ae52e94 100644 --- a/cpp/core/shuffle/Spill.cc +++ b/cpp/core/shuffle/Spill.cc @@ -62,10 +62,13 @@ void Spill::insertPayload( payloadType, numRows, isValidityBuffer, rawIs_, rawSize, pool, codec)}); break; case Payload::Type::kCompressed: + case Payload::Type::kRaw: partitionPayloads_.push_back( {partitionId, std::make_unique(numRows, isValidityBuffer, rawIs_, rawSize, pool)}); break; + default: + throw GlutenException("Unreachable."); } } diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc b/cpp/velox/shuffle/VeloxShuffleReader.cc index 22298ef91b32..c2e366c7e684 100644 --- a/cpp/velox/shuffle/VeloxShuffleReader.cc +++ b/cpp/velox/shuffle/VeloxShuffleReader.cc @@ -404,13 +404,7 @@ std::unique_ptr VeloxColumnarBatchDeserializerFactory::cr decompressTime_); } return std::make_unique( - veloxPool_, - rowType_, - batchSize_, - veloxCompressionType_, - [this](int64_t decompressionTime) { this->decompressTime_ += decompressionTime; }, - [this](int64_t deserializeTime) { this->deserializeTime_ += deserializeTime; }, - in); + veloxPool_, rowType_, batchSize_, veloxCompressionType_, deserializeTime_, std::move(in)); } VeloxShuffleReaderOutStreamWrapper::VeloxShuffleReaderOutStreamWrapper( @@ -418,20 +412,17 @@ VeloxShuffleReaderOutStreamWrapper::VeloxShuffleReaderOutStreamWrapper( const RowTypePtr& rowType, int32_t batchSize, facebook::velox::common::CompressionKind veloxCompressionType, - const std::function decompressionTimeAccumulator, - const std::function deserializeTimeAccumulator, - const std::shared_ptr in) + int64_t& deserializeTime, + std::shared_ptr in) : veloxPool_(veloxPool), rowType_(rowType), batchSize_(batchSize), veloxCompressionType_(veloxCompressionType), - decompressionTimeAccumulator_(decompressionTimeAccumulator), - deserializeTimeAccumulator_(deserializeTimeAccumulator) { + deserializeTime_(deserializeTime) { constexpr uint64_t kMaxReadBufferSize = (1 << 20) - AlignedBuffer::kPaddedSize; auto buffer = AlignedBuffer::allocate(kMaxReadBufferSize, veloxPool_.get()); in_ = std::make_unique(std::move(in), std::move(buffer)); serdeOptions_ = {false, veloxCompressionType_}; - RowVectorPtr rowVector; } std::shared_ptr VeloxShuffleReaderOutStreamWrapper::next() { @@ -439,6 +430,8 @@ std::shared_ptr VeloxShuffleReaderOutStreamWrapper::next() { return nullptr; } + ScopedTimer timer(&deserializeTime_); + RowVectorPtr rowVector; VectorStreamGroup::read(in_.get(), veloxPool_.get(), rowType_, &rowVector, &serdeOptions_); @@ -452,11 +445,6 @@ std::shared_ptr VeloxShuffleReaderOutStreamWrapper::next() { rowVector->append(rowVectorTemp.get()); } - int64_t decompressTime = 0LL; - int64_t deserializeTime = 0LL; - - decompressionTimeAccumulator_(decompressTime); - deserializeTimeAccumulator_(deserializeTime); return std::make_shared(std::move(rowVector)); } @@ -527,7 +515,7 @@ void VeloxInputStream::next(bool throwIfPastEnd) { offset_ = in_->Read(readBytes, buffer_->asMutable()).ValueOr(0); if (offset_ > 0) { int32_t realBytes = offset_; - VELOX_CHECK_LT(0, realBytes, "Reading past end of spill file"); + VELOX_CHECK_LT(0, realBytes, "Reading past end of file."); setRange({buffer_->asMutable(), realBytes, 0}); } } diff --git a/cpp/velox/shuffle/VeloxShuffleReader.h b/cpp/velox/shuffle/VeloxShuffleReader.h index 3a0d8f9ffb73..f18554db1654 100644 --- a/cpp/velox/shuffle/VeloxShuffleReader.h +++ b/cpp/velox/shuffle/VeloxShuffleReader.h @@ -80,10 +80,9 @@ class VeloxShuffleReaderOutStreamWrapper : public ColumnarBatchIterator { const std::shared_ptr& veloxPool, const facebook::velox::RowTypePtr& rowType, int32_t batchSize, - const facebook::velox::common::CompressionKind veloxCompressionType, - const std::function decompressionTimeAccumulator, - const std::function deserializeTimeAccumulator, - const std::shared_ptr in); + facebook::velox::common::CompressionKind veloxCompressionType, + int64_t& deserializeTime, + std::shared_ptr in); std::shared_ptr next(); @@ -96,8 +95,7 @@ class VeloxShuffleReaderOutStreamWrapper : public ColumnarBatchIterator { int32_t batchSize_; facebook::velox::common::CompressionKind veloxCompressionType_; facebook::velox::serializer::presto::PrestoVectorSerde::PrestoOptions serdeOptions_; - std::function decompressionTimeAccumulator_; - std::function deserializeTimeAccumulator_; + int64_t& deserializeTime_; std::shared_ptr in_; }; diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc index 85fccf7165bd..9d9dec72edd7 100644 --- a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc @@ -53,9 +53,6 @@ arrow::Status VeloxSortBasedShuffleWriter::init() { DLOG(INFO) << "Create partitioning type: " << std::to_string(options_.partitioning); rowVectorIndexMap_.reserve(numPartitions_); - for (auto pid = 0; pid < numPartitions_; ++pid) { - rowVectorIndexMap_[pid].reserve(options_.bufferSize); - } bufferOutputStream_ = std::make_unique(veloxPool_.get()); return arrow::Status::OK(); diff --git a/cpp/velox/tests/VeloxShuffleWriterTest.cc b/cpp/velox/tests/VeloxShuffleWriterTest.cc index fdf3e4491169..52649128a999 100644 --- a/cpp/velox/tests/VeloxShuffleWriterTest.cc +++ b/cpp/velox/tests/VeloxShuffleWriterTest.cc @@ -70,6 +70,8 @@ std::vector createShuffleTestParams() { std::vector mergeBufferSizes = {0, 3, 4, 10, 4096}; for (const auto& compression : compressions) { + params.push_back( + ShuffleTestParams{ShuffleWriterType::kSortShuffle, PartitionWriterType::kLocal, compression, 0, 0}); params.push_back(ShuffleTestParams{ShuffleWriterType::kSortShuffle, PartitionWriterType::kRss, compression, 0, 0}); for (const auto compressionThreshold : compressionThresholds) { for (const auto mergeBufferSize : mergeBufferSizes) { diff --git a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h index 66732c97a9a9..94e2b071b430 100644 --- a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h +++ b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h @@ -264,8 +264,7 @@ class VeloxShuffleWriterTest : public ::testing::TestWithParam The setting is based on the number of cores in your system. Use 72 cores as an example. | 0-17,36-53 |18-35,54-71 | | spark.gluten.sql.native.bloomFilter | Enable or Disable native runtime bloom filter. | true | -| spark.gluten.sql.native.arrow.reader.enabled | Enable or Disable native arrow read CSV file format | false | +| spark.gluten.sql.native.arrow.reader.enabled | Enable or Disable native arrow read CSV file format | false | | spark.gluten.sql.columnar.wholeStage.fallback.threshold | Configure the threshold for whether whole stage will fall back in AQE supported case by counting the number of ColumnarToRow & vanilla leaf node | \>= 1 | | spark.gluten.sql.columnar.query.fallback.threshold | Configure the threshold for whether query will fall back by counting the number of ColumnarToRow & vanilla leaf node | \>= 1 | | spark.gluten.sql.columnar.fallback.ignoreRowToColumnar | When true, the fallback policy ignores the RowToColumnar when counting fallback number. | true | diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala index 8d7456494a09..99b44a2de350 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala @@ -73,7 +73,9 @@ trait MetricsApi extends Serializable { def genCustomExpandMetrics(sparkContext: SparkContext): Map[String, SQLMetric] - def genColumnarShuffleExchangeMetrics(sparkContext: SparkContext): Map[String, SQLMetric] + def genColumnarShuffleExchangeMetrics( + sparkContext: SparkContext, + isSort: Boolean): Map[String, SQLMetric] def genWindowTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 69777f77a561..f41b26374a93 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -318,7 +318,8 @@ trait SparkPlanExecApi { newPartitioning: Partitioning, serializer: Serializer, writeMetrics: Map[String, SQLMetric], - metrics: Map[String, SQLMetric]): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] + metrics: Map[String, SQLMetric], + isSort: Boolean): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] /** * Generate ColumnarShuffleWriter for ColumnarShuffleManager. @@ -333,7 +334,10 @@ trait SparkPlanExecApi { * * @return */ - def createColumnarBatchSerializer(schema: StructType, metrics: Map[String, SQLMetric]): Serializer + def createColumnarBatchSerializer( + schema: StructType, + metrics: Map[String, SQLMetric], + isSort: Boolean): Serializer /** Create broadcast relation for BroadcastExchangeExec */ def createBroadcastRelation( diff --git a/gluten-core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala b/gluten-core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala index 15f403afac3d..9f9f867ff69e 100644 --- a/gluten-core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala +++ b/gluten-core/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleDependency.scala @@ -58,7 +58,8 @@ class ColumnarShuffleDependency[K: ClassTag, V: ClassTag, C: ClassTag]( override val mapSideCombine: Boolean = false, override val shuffleWriterProcessor: ShuffleWriteProcessor = new ShuffleWriteProcessor, val nativePartitioning: NativePartitioning, - val metrics: Map[String, SQLMetric]) + val metrics: Map[String, SQLMetric], + val isSort: Boolean = false) extends ShuffleDependency[K, V, C]( _rdd, partitioner, diff --git a/gluten-core/src/main/scala/org/apache/spark/shuffle/GlutenShuffleWriterWrapper.scala b/gluten-core/src/main/scala/org/apache/spark/shuffle/GlutenShuffleWriterWrapper.scala index 368fa803cccd..5170bd2b2a28 100644 --- a/gluten-core/src/main/scala/org/apache/spark/shuffle/GlutenShuffleWriterWrapper.scala +++ b/gluten-core/src/main/scala/org/apache/spark/shuffle/GlutenShuffleWriterWrapper.scala @@ -26,7 +26,8 @@ case class GenShuffleWriterParameters[K, V]( shuffleBlockResolver: IndexShuffleBlockResolver, columnarShuffleHandle: ColumnarShuffleHandle[K, V], mapId: Long, - metrics: ShuffleWriteMetricsReporter) + metrics: ShuffleWriteMetricsReporter, + isSort: Boolean = false) object GlutenShuffleWriterWrapper { @@ -34,9 +35,17 @@ object GlutenShuffleWriterWrapper { shuffleBlockResolver: IndexShuffleBlockResolver, columnarShuffleHandle: ColumnarShuffleHandle[K, V], mapId: Long, - metrics: ShuffleWriteMetricsReporter): ShuffleWriter[K, V] = + metrics: ShuffleWriteMetricsReporter): ShuffleWriter[K, V] = { + val isSort = + columnarShuffleHandle.dependency.asInstanceOf[ColumnarShuffleDependency[K, V, V]].isSort BackendsApiManager.getSparkPlanExecApiInstance .genColumnarShuffleWriter( - GenShuffleWriterParameters(shuffleBlockResolver, columnarShuffleHandle, mapId, metrics)) + GenShuffleWriterParameters( + shuffleBlockResolver, + columnarShuffleHandle, + mapId, + metrics, + isSort)) .shuffleWriter + } } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala index e2d47510a8e8..85a4dd3878a3 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.sql.execution +import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.GlutenPlan import org.apache.gluten.extension.ValidationResult @@ -52,10 +53,15 @@ case class ColumnarShuffleExchangeExec( private[sql] lazy val readMetrics = SQLColumnarShuffleReadMetricsReporter.createShuffleReadMetrics(sparkContext) + val isSortBasedShuffle: Boolean = + outputPartitioning.numPartitions > GlutenConfig.getConf.columnarShuffleSortThreshold + // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics = BackendsApiManager.getMetricsApiInstance - .genColumnarShuffleExchangeMetrics(sparkContext) ++ readMetrics ++ writeMetrics + .genColumnarShuffleExchangeMetrics( + sparkContext, + isSortBasedShuffle) ++ readMetrics ++ writeMetrics @transient lazy val inputColumnarRDD: RDD[ColumnarBatch] = child.executeColumnar() @@ -82,7 +88,8 @@ case class ColumnarShuffleExchangeExec( outputPartitioning, serializer, writeMetrics, - metrics) + metrics, + isSortBasedShuffle) } // 'shuffleDependency' is only needed when enable AQE. @@ -103,7 +110,7 @@ case class ColumnarShuffleExchangeExec( // super.stringArgs ++ Iterator(output.map(o => s"${o}#${o.dataType.simpleString}")) val serializer: Serializer = BackendsApiManager.getSparkPlanExecApiInstance - .createColumnarBatchSerializer(schema, metrics) + .createColumnarBatchSerializer(schema, metrics, isSortBasedShuffle) var cachedShuffleRDD: ShuffledColumnarBatchRDD = _ @@ -190,7 +197,8 @@ object ColumnarShuffleExchangeExec extends Logging { newPartitioning: Partitioning, serializer: Serializer, writeMetrics: Map[String, SQLMetric], - metrics: Map[String, SQLMetric]) + metrics: Map[String, SQLMetric], + isSortBasedShuffle: Boolean) // scalastyle:on argcount : ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = { BackendsApiManager.getSparkPlanExecApiInstance.genShuffleDependency( @@ -200,7 +208,8 @@ object ColumnarShuffleExchangeExec extends Logging { newPartitioning: Partitioning, serializer: Serializer, writeMetrics, - metrics) + metrics, + isSortBasedShuffle) } class DummyPairRDDWithPartitions(@transient private val sc: SparkContext, numPartitions: Int) diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java index f6dfadaafcf6..03b15dcd96de 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderMetrics.java @@ -18,7 +18,6 @@ public class ShuffleReaderMetrics { private long decompressTime; - private long ipcTime; private long deserializeTime; public void setDecompressTime(long decompressTime) { @@ -29,14 +28,6 @@ public long getDecompressTime() { return decompressTime; } - public void setIpcTime(long ipcTime) { - this.ipcTime = ipcTime; - } - - public long getIpcTime() { - return ipcTime; - } - public void setDeserializeTime(long deserializeTime) { this.deserializeTime = deserializeTime; } diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java index ed312fa14b24..f4e1172757fe 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java @@ -69,7 +69,8 @@ public long make( double reallocThreshold, long handle, long taskAttemptId, - int startPartitionId) { + int startPartitionId, + String shuffleWriterType) { return nativeMake( part.getShortName(), part.getNumPartitions(), @@ -93,7 +94,7 @@ public long make( 0, null, "local", - "hash"); + shuffleWriterType); } /** diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index 69e9aa9c951a..326f836a0a58 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -47,21 +47,23 @@ class ColumnarBatchSerializer( schema: StructType, readBatchNumRows: SQLMetric, numOutputRows: SQLMetric, - decompressTime: SQLMetric, - ipcTime: SQLMetric, - deserializeTime: SQLMetric) + deserializeTime: SQLMetric, + decompressTime: Option[SQLMetric], + isSort: Boolean) extends Serializer with Serializable { + private val shuffleWriterType = if (isSort) "sort" else "hash" + /** Creates a new [[SerializerInstance]]. */ override def newInstance(): SerializerInstance = { new ColumnarBatchSerializerInstance( schema, readBatchNumRows, numOutputRows, + deserializeTime, decompressTime, - ipcTime, - deserializeTime) + shuffleWriterType) } override def supportsRelocationOfSerializedObjects: Boolean = true @@ -71,9 +73,9 @@ private class ColumnarBatchSerializerInstance( schema: StructType, readBatchNumRows: SQLMetric, numOutputRows: SQLMetric, - decompressTime: SQLMetric, - ipcTime: SQLMetric, - deserializeTime: SQLMetric) + deserializeTime: SQLMetric, + decompressTime: Option[SQLMetric], + shuffleWriterType: String) extends SerializerInstance with Logging { @@ -103,7 +105,7 @@ private class ColumnarBatchSerializerInstance( compressionCodec, compressionCodecBackend, batchSize, - "hash" + shuffleWriterType ) // Close shuffle reader instance as lately as the end of task processing, // since the native reader could hold a reference to memory pool that @@ -113,9 +115,11 @@ private class ColumnarBatchSerializerInstance( // Collect Metrics val readerMetrics = new ShuffleReaderMetrics() jniWrapper.populateMetrics(shuffleReaderHandle, readerMetrics) - decompressTime += readerMetrics.getDecompressTime - ipcTime += readerMetrics.getIpcTime deserializeTime += readerMetrics.getDeserializeTime + decompressTime match { + case Some(t) => t += readerMetrics.getDecompressTime + case None => + } jniWrapper.close(shuffleReaderHandle) cSchema.release() diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala index c797257f1fd1..af004b54fd93 100644 --- a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala +++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala @@ -38,7 +38,8 @@ class ColumnarShuffleWriter[K, V]( shuffleBlockResolver: IndexShuffleBlockResolver, handle: BaseShuffleHandle[K, V, V], mapId: Long, - writeMetrics: ShuffleWriteMetricsReporter) + writeMetrics: ShuffleWriteMetricsReporter, + isSort: Boolean) extends ShuffleWriter[K, V] with Logging { @@ -107,6 +108,8 @@ class ColumnarShuffleWriter[K, V]( private val taskContext: TaskContext = TaskContext.get() + private val shuffleWriterType: String = if (isSort) "sort" else "hash" + private def availableOffHeapPerTask(): Long = { val perTask = SparkMemoryUtil.getCurrentAvailableOffHeapMemory / SparkResourceUtil.getTaskSlots(conf) @@ -177,7 +180,8 @@ class ColumnarShuffleWriter[K, V]( reallocThreshold, handle, taskContext.taskAttemptId(), - GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, taskContext.partitionId) + GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, taskContext.partitionId), + shuffleWriterType ) } val startTime = System.nanoTime() @@ -203,10 +207,12 @@ class ColumnarShuffleWriter[K, V]( splitResult.getTotalWriteTime - splitResult.getTotalCompressTime) dep.metrics("spillTime").add(splitResult.getTotalSpillTime) - dep.metrics("compressTime").add(splitResult.getTotalCompressTime) dep.metrics("bytesSpilled").add(splitResult.getTotalBytesSpilled) dep.metrics("splitBufferSize").add(splitResult.getSplitBufferSize) dep.metrics("dataSize").add(splitResult.getRawPartitionLengths.sum) + if (!isSort) { + dep.metrics("compressTime").add(splitResult.getTotalCompressTime) + } writeMetrics.incBytesWritten(splitResult.getTotalBytesWritten) writeMetrics.incWriteTime(splitResult.getTotalWriteTime + splitResult.getTotalSpillTime) diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/utils/ShuffleUtil.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/utils/ShuffleUtil.scala index d0589c90d6c8..7e3bf50fc81b 100644 --- a/gluten-data/src/main/scala/org/apache/spark/shuffle/utils/ShuffleUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/utils/ShuffleUtil.scala @@ -27,6 +27,7 @@ object ShuffleUtil { parameters.shuffleBlockResolver, parameters.columnarShuffleHandle, parameters.mapId, - parameters.metrics)) + parameters.metrics, + parameters.isSort)) } } diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index d3e7b409686a..083915f12db9 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -81,7 +81,8 @@ object ExecUtil { newPartitioning: Partitioning, serializer: Serializer, writeMetrics: Map[String, SQLMetric], - metrics: Map[String, SQLMetric]): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = { + metrics: Map[String, SQLMetric], + isSort: Boolean): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = { metrics("numPartitions").set(newPartitioning.numPartitions) val executionId = rdd.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) SQLMetrics.postDriverMetricUpdates( @@ -200,7 +201,8 @@ object ExecUtil { serializer, shuffleWriterProcessor = ShuffleExchangeExec.createShuffleWriteProcessor(writeMetrics), nativePartitioning = nativePartitioning, - metrics = metrics + metrics = metrics, + isSort = isSort ) dependency diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java index 8e0f4f085af5..f91141c1eb84 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java @@ -59,6 +59,8 @@ public ShuffleWriter getWriter( } RssShuffleHandle rssHandle = (RssShuffleHandle) handle; if (rssHandle.getDependency() instanceof ColumnarShuffleDependency) { + ColumnarShuffleDependency dependency = + (ColumnarShuffleDependency) rssHandle.getDependency(); setPusherAppId(rssHandle); String taskId = "" + context.taskAttemptId() + "_" + context.attemptNumber(); ShuffleWriteMetrics writeMetrics; @@ -79,7 +81,8 @@ public ShuffleWriter getWriter( shuffleWriteClient, rssHandle, this::markFailedTask, - context); + context, + dependency.isSort()); } else { return super.getWriter(handle, mapId, context, metrics); } diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index c0063c6f4274..471d70f4d1a5 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -72,6 +72,7 @@ public class VeloxUniffleColumnarShuffleWriter extends RssShuffleWriter columnarDep; private final SparkConf sparkConf; @@ -93,7 +94,8 @@ public VeloxUniffleColumnarShuffleWriter( ShuffleWriteClient shuffleWriteClient, RssShuffleHandle rssHandle, Function taskFailureCallback, - TaskContext context) { + TaskContext context, + Boolean isSort) { super( appId, shuffleId, @@ -109,6 +111,7 @@ public VeloxUniffleColumnarShuffleWriter( columnarDep = (ColumnarShuffleDependency) rssHandle.getDependency(); this.partitionId = partitionId; this.sparkConf = sparkConf; + this.isSort = isSort; bufferSize = (int) sparkConf.getSizeAsBytes( @@ -181,7 +184,7 @@ public Set applicablePhases() { GlutenShuffleUtils.getStartPartitionId( columnarDep.nativePartitioning(), partitionId), "uniffle", - "hash", + isSort ? "sort" : "hash", reallocThreshold); } long startTime = System.nanoTime(); diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 1f682557bb18..0fcaed8217ec 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -163,6 +163,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { @deprecated def broadcastCacheTimeout: Int = conf.getConf(COLUMNAR_BROADCAST_CACHE_TIMEOUT) + def columnarShuffleSortThreshold: Int = conf.getConf(COLUMNAR_SHUFFLE_SORT_THRESHOLD) + def columnarShuffleReallocThreshold: Double = conf.getConf(COLUMNAR_SHUFFLE_REALLOC_THRESHOLD) def columnarShuffleMergeThreshold: Double = conf.getConf(SHUFFLE_WRITER_MERGE_THRESHOLD) @@ -903,6 +905,14 @@ object GlutenConfig { .booleanConf .createWithDefault(true) + val COLUMNAR_SHUFFLE_SORT_THRESHOLD = + buildConf("spark.gluten.sql.columnar.shuffle.sort.threshold") + .internal() + .doc("The threshold to determine whether to use sort-based columnar shuffle. Sort-based " + + "shuffle will be used if the number of partitions is greater than this threshold.") + .intConf + .createWithDefault(100000) + val COLUMNAR_PREFER_ENABLED = buildConf("spark.gluten.sql.columnar.preferColumnar") .internal() From 5dd884e150839a6377087d302cfb8f242948bd9c Mon Sep 17 00:00:00 2001 From: James Xu Date: Thu, 23 May 2024 14:43:03 +0800 Subject: [PATCH 136/402] [GLUTEN-5757][CORE] Remove unnecessary ProjectExecTransformer for Generate (#5782) If generator function's input is already Attribute reference, we omit the introduction of the ProjectExec. Previous implementation always assume there is Project under Generate. In the new implementation we added a metadata(injectedProject) in Substrait plan to tell us whether there is a dedicated Project under Generate --- .../execution/GenerateExecTransformer.scala | 69 +++++++++++++------ .../gluten/execution/TestOperator.scala | 30 ++++++-- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 13 ++-- .../WholeStageTransformerSuite.scala | 30 +++++++- 4 files changed, 112 insertions(+), 30 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala index 23addb89ea89..c9b0abd6fabf 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala @@ -95,6 +95,21 @@ case class GenerateExecTransformer( operatorId) } + /** + * Is the specified expression an Attribute reference? + * @param expr + * @param replaceBoundReference + * @return + */ + private def isAttributeReference( + expr: Expression, + replaceBoundReference: Boolean = false): Boolean = + expr match { + case _: Attribute => true + case _: BoundReference if !replaceBoundReference => true + case _ => false + } + private def getExtensionNode(validation: Boolean): AdvancedExtensionNode = { if (!validation) { // Start with "GenerateParameters:" @@ -111,14 +126,26 @@ case class GenerateExecTransformer( .append("\n") // isStack: 1 for Stack, 0 for others. - val isStack = if (generator.isInstanceOf[Stack]) { - "1" + val isStack = generator.isInstanceOf[Stack] + parametersStr + .append("isStack=") + .append(if (isStack) "1" else "0") + .append("\n") + + val injectProject = if (isStack) { + // We always need to inject a Project for stack because we organize + // stack's flat params into arrays, e.g. stack(2, 1, 2, 3) is + // organized into two arrays: [1, 2] and [3, null]. + true } else { - "0" + // Other generator function only have one param, so we just check whether + // the only param(generator.children.head) is attribute reference or not. + !isAttributeReference(generator.children.head, true); } + parametersStr - .append("isStack=") - .append(isStack) + .append("injectedProject=") + .append(if (injectProject) "1" else "0") .append("\n") val message = StringValue @@ -158,27 +185,27 @@ object PullOutGenerateProjectHelper extends PullOutProjectHelper { val expressionMap = new mutable.HashMap[Expression, NamedExpression]() // The new child should be either the original Attribute, // or an Alias to other expressions. - val generatorAttr = replaceExpressionWithAttribute( + replaceExpressionWithAttribute( generate.generator.asInstanceOf[UnaryExpression].child, expressionMap, replaceBoundReference = true) - val newGeneratorChild = if (expressionMap.isEmpty) { - // generator.child is Attribute - generatorAttr.asInstanceOf[Attribute] + + if (!expressionMap.isEmpty) { + // generator.child is not an Attribute reference, e.g Literal/CreateArray/CreateMap. + // We plug in a Project to make it an Attribute reference. + // NOTE: DO NOT use eliminateProjectList to create the project list because + // newGeneratorChild can be a duplicated Attribute in generate.child.output. The native + // side identifies the last field of projection as generator's input. + val newGeneratorChildren = Seq(expressionMap.values.head) + generate.copy( + generator = + generate.generator.withNewChildren(newGeneratorChildren).asInstanceOf[Generator], + child = ProjectExec(generate.child.output ++ newGeneratorChildren, generate.child) + ) } else { - // generator.child is other expression, e.g Literal/CreateArray/CreateMap - expressionMap.values.head + // generator.child is Attribute, no need to introduce a Project. + generate } - val newGeneratorChildren = Seq(newGeneratorChild) - - // Avoid using eliminateProjectList to create the project list - // because newGeneratorChild can be a duplicated Attribute in generate.child.output. - // The native side identifies the last field of projection as generator's input. - generate.copy( - generator = - generate.generator.withNewChildren(newGeneratorChildren).asInstanceOf[Generator], - child = ProjectExec(generate.child.output ++ newGeneratorChildren, generate.child) - ) case stack: Stack => val numRows = stack.children.head.eval().asInstanceOf[Int] val numFields = Math.ceil((stack.children.size - 1.0) / numRows).toInt diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 287bf1e9bda2..657039572d93 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -23,9 +23,9 @@ import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, Row} -import org.apache.spark.sql.execution.{ArrowFileSourceScanExec, ColumnarToRowExec, FilterExec, GenerateExec, ProjectExec, RDDScanExec} +import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.window.WindowExec -import org.apache.spark.sql.functions.{avg, col, lit, to_date, udf} +import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DecimalType, StringType, StructField, StructType} @@ -787,7 +787,8 @@ class TestOperator extends VeloxWholeStageTransformerSuite { runQueryAndCompare(s""" |SELECT $func(a) from t2; |""".stripMargin) { - checkGlutenOperatorMatch[GenerateExecTransformer] + // No ProjectExecTransformer is introduced. + checkSparkOperatorChainMatch[GenerateExecTransformer, FilterExecTransformer] } sql("""select * from values | map(1, 'a', 2, 'b', 3, null), @@ -797,7 +798,8 @@ class TestOperator extends VeloxWholeStageTransformerSuite { runQueryAndCompare(s""" |SELECT $func(a) from t2; |""".stripMargin) { - checkGlutenOperatorMatch[GenerateExecTransformer] + // No ProjectExecTransformer is introduced. + checkSparkOperatorChainMatch[GenerateExecTransformer, FilterExecTransformer] } } } @@ -908,6 +910,26 @@ class TestOperator extends VeloxWholeStageTransformerSuite { checkGlutenOperatorMatch[GenerateExecTransformer] } } + + // More complex case which might cause projection name conflict. + withTempView("script_trans") { + sql("""SELECT * FROM VALUES + |(1, 2, 3), + |(4, 5, 6), + |(7, 8, 9) + |AS script_trans(a, b, c) + """.stripMargin).createOrReplaceTempView("script_trans") + runQueryAndCompare(s"""SELECT TRANSFORM(b, MAX(a), CAST(SUM(c) AS STRING), myCol, myCol2) + | USING 'cat' AS (a STRING, b STRING, c STRING, d ARRAY, e STRING) + |FROM script_trans + |LATERAL VIEW explode(array(array(1,2,3))) myTable AS myCol + |LATERAL VIEW explode(myTable.myCol) myTable2 AS myCol2 + |WHERE a <= 4 + |GROUP BY b, myCol, myCol2 + |HAVING max(a) > 1""".stripMargin) { + checkSparkOperatorChainMatch[GenerateExecTransformer, FilterExecTransformer] + } + } } test("test array functions") { diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 2362030661d0..b82eead2c565 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -747,12 +747,17 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: replicated.emplace_back(std::dynamic_pointer_cast(expression)); } - auto projNode = std::dynamic_pointer_cast(childNode); + auto injectedProject = generateRel.has_advanced_extension() && + SubstraitParser::configSetInOptimization(generateRel.advanced_extension(), "injectedProject="); - bool isStack = generateRel.has_advanced_extension() && - SubstraitParser::configSetInOptimization(generateRel.advanced_extension(), "isStack="); + if (injectedProject) { + auto projNode = std::dynamic_pointer_cast(childNode); + VELOX_CHECK( + projNode != nullptr && projNode->names().size() > requiredChildOutput.size(), + "injectedProject is true, but the Project is missing or does not have the corresponding projection field") - if (projNode != nullptr && projNode->names().size() > requiredChildOutput.size()) { + bool isStack = generateRel.has_advanced_extension() && + SubstraitParser::configSetInOptimization(generateRel.advanced_extension(), "isStack="); // Generator function's input is NOT a field reference. if (!isStack) { // For generator function which is not stack, e.g. explode(array(1,2,3)), a sample diff --git a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index 5f60de27b415..7d2d48828fb3 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -24,7 +24,7 @@ import org.apache.gluten.utils.Arm import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.{DataFrame, GlutenQueryTest, Row} -import org.apache.spark.sql.execution.{CommandResultExec, SparkPlan} +import org.apache.spark.sql.execution.{CommandResultExec, SparkPlan, UnaryExecNode} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, ShuffleQueryStageExec} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.DoubleType @@ -254,6 +254,34 @@ abstract class WholeStageTransformerSuite assert(executedPlan.exists(plan => tag.runtimeClass.isInstance(plan))) } + /** + * Check whether the executed plan of a dataframe contains the expected plan chain. + * + * @param df + * : the input dataframe. + * @param tag + * : class of the expected plan. + * @param childTag + * : class of the expected plan's child. + * @tparam T + * : type of the expected plan. + * @tparam PT + * : type of the expected plan's child. + */ + def checkSparkOperatorChainMatch[T <: UnaryExecNode, PT <: UnaryExecNode]( + df: DataFrame)(implicit tag: ClassTag[T], childTag: ClassTag[PT]): Unit = { + val executedPlan = getExecutedPlan(df) + assert( + executedPlan.exists( + plan => + tag.runtimeClass.isInstance(plan) + && childTag.runtimeClass.isInstance(plan.children.head)), + s"Expect an operator chain of [${tag.runtimeClass.getSimpleName} ->" + + s"${childTag.runtimeClass.getSimpleName}] exists in executedPlan: \n" + + s"${executedPlan.last}" + ) + } + /** * run a query with native engine as well as vanilla spark then compare the result set for * correctness check From fb4cb3cb811140b082c250d027c0ce65d170a79f Mon Sep 17 00:00:00 2001 From: Rui Mo Date: Thu, 23 May 2024 16:58:39 +0800 Subject: [PATCH 137/402] [VL] Daily Update Velox Version (2024_05_23) (#5847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream Velox's New Commits: ``` 96c51ae1b by Pedro Eugenio Rocha Pedreira, Re-enable alternative plans in join fuzzer (#9898) 8f5d3466f by Kk Pulla, Add APIs to get simple and vector function signatures in function registry (#9884) 80235488b by Wei He, Fix timeout of SharedArbitrationTest.skipNonReclaimableTaskTest (#9894) ffc781e93 by Bikramjeet Vig, Fix build failure due to clashing of std and folly's versions of identity functor (#9891) 47fc3dda4 by NEUpanning, Add allowOverflow flag to Timestamp::toTimezone (#9836) 675b6dd76 by Bikramjeet Vig, Fix NaN handling for array_position (#9832) 2058231e3 by Bikramjeet Vig, Fix NaN handling for array_remove and contains UDFs (#9807) fbb357807 by Masha Basmanova, Introduce velox::Expected (#9858) 890c0b457 by Masha Basmanova, Optimize TRY_CAST(varchar as integer) for invalid strings. (#9872) 2730f95ae by 高阳阳, Add width_bucket Spark function (#9743) 60b9b6e16 by Deepak Majeti, Remove unused headers in base/Exceptions.h (#9344) 6f329e9de by Wei He, Fix flaky test of SharedArbitrationTest.skipNonReclaimableTaskTest (#9883) feb98f6c9 by Bikramjeet Vig, Fix NaN handling for multiple array UDFs (#9797) 206ff47d9 by Sandino Flores, Update cmake to version 3.28.3 (#9861) ``` --- .github/workflows/velox_docker.yml | 22 +++++++++++++--------- .github/workflows/velox_docker_cache.yml | 2 +- ep/build-velox/src/get_velox.sh | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index d7644b5d0be5..9b00535b474d 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -496,7 +496,7 @@ jobs: build-native-lib-centos-8: runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 steps: - uses: actions/checkout@v2 - name: Generate cache key @@ -542,7 +542,7 @@ jobs: run-spark-test-spark32: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -597,6 +597,7 @@ jobs: mv sql shims/spark32/spark_home/ && \ dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools && \ pip3 install pyspark==3.2.2 cython && \ pip3 install pandas pyarrow - name: Build and run unit test for Spark 3.2.2 (other tests) @@ -620,7 +621,7 @@ jobs: run-spark-test-spark32-slow: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -666,7 +667,7 @@ jobs: run-spark-test-spark33: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -717,6 +718,7 @@ jobs: mv sql shims/spark33/spark_home/ && \ dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools && \ pip3 install pyspark==3.3.1 cython && \ pip3 install pandas pyarrow - name: Build and Run unit test for Spark 3.3.1 (other tests) @@ -736,7 +738,7 @@ jobs: run-spark-test-spark33-slow: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -782,7 +784,7 @@ jobs: run-spark-test-spark34: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -833,6 +835,7 @@ jobs: mv sql shims/spark34/spark_home/ && \ dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools && \ pip3 install pyspark==3.4.2 cython && \ pip3 install pandas pyarrow - name: Build and Run unit test for Spark 3.4.2 (other tests) @@ -852,7 +855,7 @@ jobs: run-spark-test-spark34-slow: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -898,7 +901,7 @@ jobs: run-spark-test-spark35: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: @@ -949,6 +952,7 @@ jobs: mv sql shims/spark35/spark_home/ && \ dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install setuptools && \ pip3 install pyspark==3.5.1 cython && \ pip3 install pandas pyarrow - name: Build and Run unit test for Spark 3.5.1 (other tests) @@ -967,7 +971,7 @@ jobs: run-spark-test-spark35-slow: needs: build-native-lib-centos-8 runs-on: ubuntu-20.04 - container: ghcr.io/facebookincubator/velox-dev:circleci-avx + container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 51b0bae2f407..7d3fc96ad804 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -88,7 +88,7 @@ jobs: # runs-on: ubuntu-20.04 # env: # CCACHE_DIR: "${{ github.workspace }}/.ccache" -# container: ghcr.io/facebookincubator/velox-dev:circleci-avx +# container: ghcr.io/facebookincubator/velox-dev:centos8 # steps: # - uses: actions/checkout@v2 # - name: Setup java and maven diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index fbb0f706742f..af032e186637 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_22 +VELOX_BRANCH=2024_05_23 VELOX_HOME="" #Set on run gluten on HDFS From fa9913b9812d12ee925f5872f0dadc829810cb87 Mon Sep 17 00:00:00 2001 From: Yuan Date: Thu, 23 May 2024 17:29:53 +0800 Subject: [PATCH 138/402] [VL][CI] update the docker image in nightly cache job (#5855) update the image in cache job Signed-off-by: Yuan Zhou --- .github/workflows/velox_docker_cache.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 7d3fc96ad804..591be26093ee 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -27,7 +27,7 @@ concurrency: jobs: cache-native-lib: runs-on: ubuntu-20.04 - container: inteldpo/gluten-centos-packaging:latest # centos7 with dependencies installed + container: apache/gluten:gluten-vcpkg-builder_2024_05_22 # centos7 with dependencies installed steps: - uses: actions/checkout@v2 - name: Generate cache key From 33eadbfa62e988aeef43c1c888abf9b601de4cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Thu, 23 May 2024 20:10:54 +0800 Subject: [PATCH 139/402] [VL] Enable rand function (#5829) --- .../clickhouse/CHSparkPlanExecApi.scala | 7 - .../ScalarFunctionsValidateSuite.scala | 6 + .../functions/RegistrationAllFunctions.cc | 4 + cpp/velox/substrait/SubstraitParser.cc | 3 + docs/velox-backend-support-progress.md | 2 + .../gluten/backendsapi/SparkPlanExecApi.scala | 7 - .../expression/ExpressionConverter.scala | 5 - .../UnaryExpressionTransformer.scala | 22 - .../sql-tests/inputs/group-by-ordinal.sql | 96 ++ .../resources/sql-tests/inputs/random.sql | 17 + .../results/group-by-ordinal.sql.out | 398 +++++++ .../sql-tests/results/group-by.sql.out | 2 +- .../sql-tests/results/random.sql.out | 84 ++ .../sql-tests/inputs/group-by-ordinal.sql | 96 ++ .../resources/sql-tests/inputs/random.sql | 17 + .../results/group-by-ordinal.sql.out | 398 +++++++ .../sql-tests/results/group-by.sql.out | 2 +- .../sql-tests/results/random.sql.out | 84 ++ .../sql-tests/inputs/group-by-ordinal.sql | 96 ++ .../resources/sql-tests/inputs/random.sql | 17 + .../results/group-by-ordinal.sql.out | 523 +++++++++ .../sql-tests/results/group-by.sql.out | 2 +- .../sql-tests/results/random.sql.out | 115 ++ .../sql-tests/inputs/group-by-ordinal.sql | 96 ++ .../resources/sql-tests/inputs/random.sql | 17 + .../inputs/table-valued-functions.sql | 126 ++ .../results/group-by-ordinal.sql.out | 524 +++++++++ .../sql-tests/results/group-by.sql.out | 2 +- .../sql-tests/results/random.sql.out | 115 ++ .../results/table-valued-functions.sql.out | 1017 +++++++++++++++++ .../utils/velox/VeloxTestSettings.scala | 1 + .../sql/GlutenGeneratorFunctionSuite.scala | 11 +- 32 files changed, 3866 insertions(+), 46 deletions(-) create mode 100644 gluten-ut/spark32/src/test/resources/sql-tests/inputs/group-by-ordinal.sql create mode 100644 gluten-ut/spark32/src/test/resources/sql-tests/inputs/random.sql create mode 100644 gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out create mode 100644 gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out create mode 100644 gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql create mode 100644 gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql create mode 100644 gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out create mode 100644 gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out create mode 100644 gluten-ut/spark34/src/test/resources/sql-tests/inputs/group-by-ordinal.sql create mode 100644 gluten-ut/spark34/src/test/resources/sql-tests/inputs/random.sql create mode 100644 gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out create mode 100644 gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out create mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/inputs/group-by-ordinal.sql create mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/inputs/random.sql create mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/inputs/table-valued-functions.sql create mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/results/group-by-ordinal.sql.out create mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/results/random.sql.out create mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/results/table-valued-functions.sql.out diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 6a154cd945e0..8c2b20db6f84 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -387,13 +387,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { original: GetMapValue): ExpressionTransformer = GetMapValueTransformer(substraitExprName, left, right, original.failOnError, original) - override def genRandTransformer( - substraitExprName: String, - explicitSeed: ExpressionTransformer, - original: Rand): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(explicitSeed), original) - } - /** * Generate ShuffleDependency for ColumnarShuffleExchangeExec. * diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index b3753ab8352c..3180842adb16 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -717,6 +717,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("Test rand function") { + runQueryAndCompare("""SELECT rand() from lineitem limit 100""".stripMargin, false) { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("regexp_replace") { runQueryAndCompare( "SELECT regexp_replace(c_comment, '\\w', 'something') FROM customer limit 50") { diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index 5a6b0f6aa2e7..b88d781b69b2 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -27,6 +27,7 @@ #include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" #include "velox/functions/sparksql/Bitwise.h" #include "velox/functions/sparksql/Hash.h" +#include "velox/functions/sparksql/Rand.h" #include "velox/functions/sparksql/Register.h" #include "velox/functions/sparksql/aggregates/Register.h" #include "velox/functions/sparksql/window/WindowFunctionsRegistration.h" @@ -43,6 +44,9 @@ void registerFunctionOverwrite() { velox::registerFunction({"round"}); velox::registerFunction({"round"}); velox::registerFunction({"round"}); + // TODO: the below rand function registry can be removed after presto function registry is removed. + velox::registerFunction>({"spark_rand"}); + velox::registerFunction>({"spark_rand"}); auto kRowConstructorWithNull = RowConstructorWithNullCallToSpecialForm::kRowConstructorWithNull; velox::exec::registerVectorFunction( diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 6f221b78e9ac..f417618d8117 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -392,6 +392,9 @@ std::unordered_map SubstraitParser::substraitVeloxFunc {"bitwise_not", "spark_bitwise_not"}, {"bitwise_or", "spark_bitwise_or"}, {"bitwise_xor", "spark_bitwise_xor"}, + // TODO: the below registry for rand functions can be removed + // after presto function registry is removed. + {"rand", "spark_rand"}, {"murmur3hash", "hash_with_seed"}, {"xxhash64", "xxhash64_with_seed"}, {"modulus", "remainder"}, diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index fb68740c7a22..ccb253c24b48 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -449,3 +449,5 @@ Gluten supports 199 functions. (Drag to right to see all data types) | stack | | | S | | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | S | | xxhash64 | xxhash64 | xxhash64 | | | | | | | | | | | | | | | | | | | | | | uuid | uuid | uuid | S | | | | | | | | | | | | | | | | | | | | +| rand | rand | rand | S | | | | | | | | | | | | | | | | | | | | + diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index f41b26374a93..429b926cdceb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -165,13 +165,6 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(srcExpr, regexExpr, limitExpr), original) } - def genRandTransformer( - substraitExprName: String, - explicitSeed: ExpressionTransformer, - original: Rand): ExpressionTransformer = { - RandTransformer(substraitExprName, explicitSeed, original) - } - /** Generate an expression transformer to transform GetMapValue to Substrait. */ def genGetMapValueTransformer( substraitExprName: String, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 6e1427e2fda9..2d514118ac30 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -453,11 +453,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { LiteralTransformer(m.nullOnOverflow)), m ) - case rand: Rand => - BackendsApiManager.getSparkPlanExecApiInstance.genRandTransformer( - substraitExprName, - replaceWithExpressionTransformerInternal(rand.child, attributeSeq, expressionsMap), - rand) case _: NormalizeNaNAndZero | _: PromotePrecision | _: TaggingExpression => ChildTransformer( substraitExprName, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala index 27f8395254fd..bcbac60dec0d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala @@ -91,28 +91,6 @@ case class CheckOverflowTransformer( } } -/** - * User can specify a seed for this function. If lacked, spark will generate a random number as - * seed. We also need to pass a unique partitionIndex provided by framework to native library for - * each thread. Then, seed plus partitionIndex will be the actual seed for generator, similar to - * vanilla spark. This is based on the fact that partitioning is deterministic and one partition is - * corresponding to one task thread. - */ -case class RandTransformer( - substraitExprName: String, - explicitSeed: ExpressionTransformer, - original: Rand) - extends LeafExpressionTransformer { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - if (!original.hideSeed) { - // TODO: for user-specified seed, we need to pass partition index to native engine. - throw new GlutenNotSupportException("User-specified seed is not supported.") - } - super.doTransform(args) - } -} - case class GetStructFieldTransformer( substraitExprName: String, child: ExpressionTransformer, diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/gluten-ut/spark32/src/test/resources/sql-tests/inputs/group-by-ordinal.sql new file mode 100644 index 000000000000..b773396c050d --- /dev/null +++ b/gluten-ut/spark32/src/test/resources/sql-tests/inputs/group-by-ordinal.sql @@ -0,0 +1,96 @@ +-- group by ordinal positions + +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b); + +-- basic case +select a, sum(b) from data group by 1; + +-- constant case +select 1, 2, sum(b) from data group by 1, 2; + +-- duplicate group by column +select a, 1, sum(b) from data group by a, 1; +select a, 1, sum(b) from data group by 1, 2; + +-- group by a non-aggregate expression's ordinal +select a, b + 2, count(2) from data group by a, 2; + +-- with alias +select a as aa, b + 2 as bb, count(2) from data group by 1, 2; + +-- foldable non-literal: this should be the same as no grouping. +select sum(b) from data group by 1 + 0; + +-- negative cases: ordinal out of range +select a, b from data group by -1; +select a, b from data group by 0; +select a, b from data group by 3; + +-- negative case: position is an aggregate expression +select a, b, sum(b) from data group by 3; +select a, b, sum(b) + 2 from data group by 3; + +-- negative case: nondeterministic expression +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2; + +-- negative case: star +select * from data group by a, b, 1; + +-- group by ordinal followed by order by +select a, count(a) from (select 1 as a) tmp group by 1 order by 1; + +-- group by ordinal followed by having +select count(a), a from (select 1 as a) tmp group by 2 having a > 0; + +-- mixed cases: group-by ordinals and aliases +select a, a AS k, count(b) from data group by k, 1; + +-- can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, 2); + +-- mixed cases: can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, b); + +-- can use ordinal with cube +select a, b, count(1) from data group by 1, 2 with cube; + +-- can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, 2); + +-- mixed cases: can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, b); + +-- can use ordinal with rollup +select a, b, count(1) from data group by 1, 2 with rollup; + +-- can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)); + +-- mixed cases: can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)); + +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)); + +-- range error +select a, b, count(1) from data group by a, -1; + +select a, b, count(1) from data group by a, 3; + +select a, b, count(1) from data group by cube(-1, 2); + +select a, b, count(1) from data group by cube(1, 3); + +-- turn off group by ordinal +set spark.sql.groupByOrdinal=false; + +-- can now group by negative literal +select sum(b) from data group by -1; diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/inputs/random.sql b/gluten-ut/spark32/src/test/resources/sql-tests/inputs/random.sql new file mode 100644 index 000000000000..a1aae7b8759d --- /dev/null +++ b/gluten-ut/spark32/src/test/resources/sql-tests/inputs/random.sql @@ -0,0 +1,17 @@ +-- rand with the seed 0 +SELECT rand(0); +SELECT rand(cast(3 / 7 AS int)); +SELECT rand(NULL); +SELECT rand(cast(NULL AS int)); + +-- rand unsupported data type +SELECT rand(1.0); + +-- randn with the seed 0 +SELECT randn(0L); +SELECT randn(cast(3 / 7 AS long)); +SELECT randn(NULL); +SELECT randn(cast(NULL AS long)); + +-- randn unsupported data type +SELECT rand('1') diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out new file mode 100644 index 000000000000..cc20dd33e0b2 --- /dev/null +++ b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -0,0 +1,398 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 33 + + +-- !query +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b) +-- !query schema +struct<> +-- !query output + + + +-- !query +select a, sum(b) from data group by 1 +-- !query schema +struct +-- !query output +1 3 +2 3 +3 3 + + +-- !query +select 1, 2, sum(b) from data group by 1, 2 +-- !query schema +struct<1:int,2:int,sum(b):bigint> +-- !query output +1 2 9 + + +-- !query +select a, 1, sum(b) from data group by a, 1 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, 1, sum(b) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, b + 2, count(2) from data group by a, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select a as aa, b + 2 as bb, count(2) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select sum(b) from data group by 1 + 0 +-- !query schema +struct +-- !query output +9 + + +-- !query +select a, b from data group by -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position -1 is not in select list (valid range is [1, 2]); line 1 pos 31 + + +-- !query +select a, b from data group by 0 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position 0 is not in select list (valid range is [1, 2]); line 1 pos 31 + + +-- !query +select a, b from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position 3 is not in select list (valid range is [1, 2]); line 1 pos 31 + + +-- !query +select a, b, sum(b) from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got sum(data.b) AS `sum(b)`; line 1 pos 39 + + +-- !query +select a, b, sum(b) + 2 from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got (sum(data.b) + CAST(2 AS BIGINT)) AS `(sum(b) + 2)`; line 1 pos 43 + + +-- !query +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2 +-- !query schema +struct +-- !query output +1 0.5488135024422883 1 +1 0.7151893651681639 2 +2 0.5448831775801376 2 +2 0.6027633705776989 1 +3 0.4236547969336536 1 +3 0.6458941151817286 2 + + +-- !query +select * from data group by a, b, 1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +Star (*) is not allowed in select list when GROUP BY ordinal position is used + + +-- !query +select a, count(a) from (select 1 as a) tmp group by 1 order by 1 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select count(a), a from (select 1 as a) tmp group by 2 having a > 0 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select a, a AS k, count(b) from data group by k, 1 +-- !query schema +struct +-- !query output +1 1 2 +2 2 2 +3 3 2 + + +-- !query +select a, b, count(1) from data group by cube(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by cube(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with cube +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with rollup +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 44 + + +-- !query +select a, b, count(1) from data group by a, 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 44 + + +-- !query +select a, b, count(1) from data group by cube(-1, 2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 46 + + +-- !query +select a, b, count(1) from data group by cube(1, 3) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 49 + + +-- !query +set spark.sql.groupByOrdinal=false +-- !query schema +struct +-- !query output +spark.sql.groupByOrdinal false + + +-- !query +select sum(b) from data group by -1 +-- !query schema +struct +-- !query output +9 diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out index a12e830c1117..b54621f8ec0c 100644 --- a/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark32/src/test/resources/sql-tests/results/group-by.sql.out @@ -662,5 +662,5 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.7604953758285915 7 +0.5488135024422883 7 1.0 2 diff --git a/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out new file mode 100644 index 000000000000..a9d334e7f2b1 --- /dev/null +++ b/gluten-ut/spark32/src/test/resources/sql-tests/results/random.sql.out @@ -0,0 +1,84 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 10 + + +-- !query +SELECT rand(0) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(3 / 7 AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(NULL) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(NULL AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(1.0) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +cannot resolve 'rand(1.0BD)' due to data type mismatch: argument 1 requires (int or bigint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7 + + +-- !query +SELECT randn(0L) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(3 / 7 AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(NULL) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(NULL AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT rand('1') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +cannot resolve 'rand('1')' due to data type mismatch: argument 1 requires (int or bigint) type, however, ''1'' is of string type.; line 1 pos 7 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql new file mode 100644 index 000000000000..b773396c050d --- /dev/null +++ b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/group-by-ordinal.sql @@ -0,0 +1,96 @@ +-- group by ordinal positions + +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b); + +-- basic case +select a, sum(b) from data group by 1; + +-- constant case +select 1, 2, sum(b) from data group by 1, 2; + +-- duplicate group by column +select a, 1, sum(b) from data group by a, 1; +select a, 1, sum(b) from data group by 1, 2; + +-- group by a non-aggregate expression's ordinal +select a, b + 2, count(2) from data group by a, 2; + +-- with alias +select a as aa, b + 2 as bb, count(2) from data group by 1, 2; + +-- foldable non-literal: this should be the same as no grouping. +select sum(b) from data group by 1 + 0; + +-- negative cases: ordinal out of range +select a, b from data group by -1; +select a, b from data group by 0; +select a, b from data group by 3; + +-- negative case: position is an aggregate expression +select a, b, sum(b) from data group by 3; +select a, b, sum(b) + 2 from data group by 3; + +-- negative case: nondeterministic expression +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2; + +-- negative case: star +select * from data group by a, b, 1; + +-- group by ordinal followed by order by +select a, count(a) from (select 1 as a) tmp group by 1 order by 1; + +-- group by ordinal followed by having +select count(a), a from (select 1 as a) tmp group by 2 having a > 0; + +-- mixed cases: group-by ordinals and aliases +select a, a AS k, count(b) from data group by k, 1; + +-- can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, 2); + +-- mixed cases: can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, b); + +-- can use ordinal with cube +select a, b, count(1) from data group by 1, 2 with cube; + +-- can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, 2); + +-- mixed cases: can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, b); + +-- can use ordinal with rollup +select a, b, count(1) from data group by 1, 2 with rollup; + +-- can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)); + +-- mixed cases: can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)); + +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)); + +-- range error +select a, b, count(1) from data group by a, -1; + +select a, b, count(1) from data group by a, 3; + +select a, b, count(1) from data group by cube(-1, 2); + +select a, b, count(1) from data group by cube(1, 3); + +-- turn off group by ordinal +set spark.sql.groupByOrdinal=false; + +-- can now group by negative literal +select sum(b) from data group by -1; diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql new file mode 100644 index 000000000000..a1aae7b8759d --- /dev/null +++ b/gluten-ut/spark33/src/test/resources/sql-tests/inputs/random.sql @@ -0,0 +1,17 @@ +-- rand with the seed 0 +SELECT rand(0); +SELECT rand(cast(3 / 7 AS int)); +SELECT rand(NULL); +SELECT rand(cast(NULL AS int)); + +-- rand unsupported data type +SELECT rand(1.0); + +-- randn with the seed 0 +SELECT randn(0L); +SELECT randn(cast(3 / 7 AS long)); +SELECT randn(NULL); +SELECT randn(cast(NULL AS long)); + +-- randn unsupported data type +SELECT rand('1') diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out new file mode 100644 index 000000000000..cc20dd33e0b2 --- /dev/null +++ b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -0,0 +1,398 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 33 + + +-- !query +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b) +-- !query schema +struct<> +-- !query output + + + +-- !query +select a, sum(b) from data group by 1 +-- !query schema +struct +-- !query output +1 3 +2 3 +3 3 + + +-- !query +select 1, 2, sum(b) from data group by 1, 2 +-- !query schema +struct<1:int,2:int,sum(b):bigint> +-- !query output +1 2 9 + + +-- !query +select a, 1, sum(b) from data group by a, 1 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, 1, sum(b) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, b + 2, count(2) from data group by a, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select a as aa, b + 2 as bb, count(2) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select sum(b) from data group by 1 + 0 +-- !query schema +struct +-- !query output +9 + + +-- !query +select a, b from data group by -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position -1 is not in select list (valid range is [1, 2]); line 1 pos 31 + + +-- !query +select a, b from data group by 0 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position 0 is not in select list (valid range is [1, 2]); line 1 pos 31 + + +-- !query +select a, b from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position 3 is not in select list (valid range is [1, 2]); line 1 pos 31 + + +-- !query +select a, b, sum(b) from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got sum(data.b) AS `sum(b)`; line 1 pos 39 + + +-- !query +select a, b, sum(b) + 2 from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got (sum(data.b) + CAST(2 AS BIGINT)) AS `(sum(b) + 2)`; line 1 pos 43 + + +-- !query +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2 +-- !query schema +struct +-- !query output +1 0.5488135024422883 1 +1 0.7151893651681639 2 +2 0.5448831775801376 2 +2 0.6027633705776989 1 +3 0.4236547969336536 1 +3 0.6458941151817286 2 + + +-- !query +select * from data group by a, b, 1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +Star (*) is not allowed in select list when GROUP BY ordinal position is used + + +-- !query +select a, count(a) from (select 1 as a) tmp group by 1 order by 1 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select count(a), a from (select 1 as a) tmp group by 2 having a > 0 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select a, a AS k, count(b) from data group by k, 1 +-- !query schema +struct +-- !query output +1 1 2 +2 2 2 +3 3 2 + + +-- !query +select a, b, count(1) from data group by cube(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by cube(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with cube +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with rollup +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 44 + + +-- !query +select a, b, count(1) from data group by a, 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 44 + + +-- !query +select a, b, count(1) from data group by cube(-1, 2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY position -1 is not in select list (valid range is [1, 3]); line 1 pos 46 + + +-- !query +select a, b, count(1) from data group by cube(1, 3) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +GROUP BY 3 refers to an expression that is or contains an aggregate function. Aggregate functions are not allowed in GROUP BY, but got count(1) AS `count(1)`; line 1 pos 49 + + +-- !query +set spark.sql.groupByOrdinal=false +-- !query schema +struct +-- !query output +spark.sql.groupByOrdinal false + + +-- !query +select sum(b) from data group by -1 +-- !query schema +struct +-- !query output +9 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out index 48b35bf1e0d8..408b9f9425bf 100644 --- a/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark33/src/test/resources/sql-tests/results/group-by.sql.out @@ -691,7 +691,7 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.7604953758285915 7 +0.5488135024422883 7 1.0 2 diff --git a/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out new file mode 100644 index 000000000000..a9d334e7f2b1 --- /dev/null +++ b/gluten-ut/spark33/src/test/resources/sql-tests/results/random.sql.out @@ -0,0 +1,84 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 10 + + +-- !query +SELECT rand(0) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(3 / 7 AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(NULL) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(NULL AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(1.0) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +cannot resolve 'rand(1.0BD)' due to data type mismatch: argument 1 requires (int or bigint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7 + + +-- !query +SELECT randn(0L) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(3 / 7 AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(NULL) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(NULL AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT rand('1') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +cannot resolve 'rand('1')' due to data type mismatch: argument 1 requires (int or bigint) type, however, ''1'' is of string type.; line 1 pos 7 diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/gluten-ut/spark34/src/test/resources/sql-tests/inputs/group-by-ordinal.sql new file mode 100644 index 000000000000..b773396c050d --- /dev/null +++ b/gluten-ut/spark34/src/test/resources/sql-tests/inputs/group-by-ordinal.sql @@ -0,0 +1,96 @@ +-- group by ordinal positions + +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b); + +-- basic case +select a, sum(b) from data group by 1; + +-- constant case +select 1, 2, sum(b) from data group by 1, 2; + +-- duplicate group by column +select a, 1, sum(b) from data group by a, 1; +select a, 1, sum(b) from data group by 1, 2; + +-- group by a non-aggregate expression's ordinal +select a, b + 2, count(2) from data group by a, 2; + +-- with alias +select a as aa, b + 2 as bb, count(2) from data group by 1, 2; + +-- foldable non-literal: this should be the same as no grouping. +select sum(b) from data group by 1 + 0; + +-- negative cases: ordinal out of range +select a, b from data group by -1; +select a, b from data group by 0; +select a, b from data group by 3; + +-- negative case: position is an aggregate expression +select a, b, sum(b) from data group by 3; +select a, b, sum(b) + 2 from data group by 3; + +-- negative case: nondeterministic expression +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2; + +-- negative case: star +select * from data group by a, b, 1; + +-- group by ordinal followed by order by +select a, count(a) from (select 1 as a) tmp group by 1 order by 1; + +-- group by ordinal followed by having +select count(a), a from (select 1 as a) tmp group by 2 having a > 0; + +-- mixed cases: group-by ordinals and aliases +select a, a AS k, count(b) from data group by k, 1; + +-- can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, 2); + +-- mixed cases: can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, b); + +-- can use ordinal with cube +select a, b, count(1) from data group by 1, 2 with cube; + +-- can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, 2); + +-- mixed cases: can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, b); + +-- can use ordinal with rollup +select a, b, count(1) from data group by 1, 2 with rollup; + +-- can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)); + +-- mixed cases: can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)); + +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)); + +-- range error +select a, b, count(1) from data group by a, -1; + +select a, b, count(1) from data group by a, 3; + +select a, b, count(1) from data group by cube(-1, 2); + +select a, b, count(1) from data group by cube(1, 3); + +-- turn off group by ordinal +set spark.sql.groupByOrdinal=false; + +-- can now group by negative literal +select sum(b) from data group by -1; diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/inputs/random.sql b/gluten-ut/spark34/src/test/resources/sql-tests/inputs/random.sql new file mode 100644 index 000000000000..a1aae7b8759d --- /dev/null +++ b/gluten-ut/spark34/src/test/resources/sql-tests/inputs/random.sql @@ -0,0 +1,17 @@ +-- rand with the seed 0 +SELECT rand(0); +SELECT rand(cast(3 / 7 AS int)); +SELECT rand(NULL); +SELECT rand(cast(NULL AS int)); + +-- rand unsupported data type +SELECT rand(1.0); + +-- randn with the seed 0 +SELECT randn(0L); +SELECT randn(cast(3 / 7 AS long)); +SELECT randn(NULL); +SELECT randn(cast(NULL AS long)); + +-- randn unsupported data type +SELECT rand('1') diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out new file mode 100644 index 000000000000..5b8637012e57 --- /dev/null +++ b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -0,0 +1,523 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b) +-- !query schema +struct<> +-- !query output + + + +-- !query +select a, sum(b) from data group by 1 +-- !query schema +struct +-- !query output +1 3 +2 3 +3 3 + + +-- !query +select 1, 2, sum(b) from data group by 1, 2 +-- !query schema +struct<1:int,2:int,sum(b):bigint> +-- !query output +1 2 9 + + +-- !query +select a, 1, sum(b) from data group by a, 1 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, 1, sum(b) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, b + 2, count(2) from data group by a, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select a as aa, b + 2 as bb, count(2) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select sum(b) from data group by 1 + 0 +-- !query schema +struct +-- !query output +9 + + +-- !query +select a, b from data group by -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "-1", + "size" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 32, + "stopIndex" : 33, + "fragment" : "-1" + } ] +} + + +-- !query +select a, b from data group by 0 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "0", + "size" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 32, + "stopIndex" : 32, + "fragment" : "0" + } ] +} + + +-- !query +select a, b from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "3", + "size" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 32, + "stopIndex" : 32, + "fragment" : "3" + } ] +} + + +-- !query +select a, b, sum(b) from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "sum(data.b) AS `sum(b)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 40, + "stopIndex" : 40, + "fragment" : "3" + } ] +} + + +-- !query +select a, b, sum(b) + 2 from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "(sum(data.b) + CAST(2 AS BIGINT)) AS `(sum(b) + 2)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 44, + "stopIndex" : 44, + "fragment" : "3" + } ] +} + + +-- !query +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2 +-- !query schema +struct +-- !query output +1 0.5488135024422883 1 +1 0.7151893651681639 2 +2 0.5448831775801376 2 +2 0.6027633705776989 1 +3 0.4236547969336536 1 +3 0.6458941151817286 2 + +-- !query +select * from data group by a, b, 1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "STAR_GROUP_BY_POS", + "sqlState" : "0A000" +} + + +-- !query +select a, count(a) from (select 1 as a) tmp group by 1 order by 1 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select count(a), a from (select 1 as a) tmp group by 2 having a > 0 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select a, a AS k, count(b) from data group by k, 1 +-- !query schema +struct +-- !query output +1 1 2 +2 2 2 +3 3 2 + + +-- !query +select a, b, count(1) from data group by cube(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by cube(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with cube +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with rollup +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "-1", + "size" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 45, + "stopIndex" : 46, + "fragment" : "-1" + } ] +} + + +-- !query +select a, b, count(1) from data group by a, 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "count(1) AS `count(1)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 45, + "stopIndex" : 45, + "fragment" : "3" + } ] +} + + +-- !query +select a, b, count(1) from data group by cube(-1, 2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "-1", + "size" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 47, + "stopIndex" : 48, + "fragment" : "-1" + } ] +} + + +-- !query +select a, b, count(1) from data group by cube(1, 3) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "count(1) AS `count(1)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 50, + "stopIndex" : 50, + "fragment" : "3" + } ] +} + + +-- !query +set spark.sql.groupByOrdinal=false +-- !query schema +struct +-- !query output +spark.sql.groupByOrdinal false + + +-- !query +select sum(b) from data group by -1 +-- !query schema +struct +-- !query output +9 diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out index f56420926050..a4a3f76fa6a7 100644 --- a/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark34/src/test/resources/sql-tests/results/group-by.sql.out @@ -907,7 +907,7 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.7604953758285915 7 +0.5488135024422883 7 1.0 2 diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out new file mode 100644 index 000000000000..f8460c1d437e --- /dev/null +++ b/gluten-ut/spark34/src/test/resources/sql-tests/results/random.sql.out @@ -0,0 +1,115 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT rand(0) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(3 / 7 AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(NULL) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(NULL AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(1.0) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"1.0\"", + "inputType" : "\"DECIMAL(2,1)\"", + "paramIndex" : "1", + "requiredType" : "(\"INT\" or \"BIGINT\")", + "sqlExpr" : "\"rand(1.0)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 16, + "fragment" : "rand(1.0)" + } ] +} + + +-- !query +SELECT randn(0L) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(3 / 7 AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(NULL) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(NULL AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT rand('1') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"1\"", + "inputType" : "\"STRING\"", + "paramIndex" : "1", + "requiredType" : "(\"INT\" or \"BIGINT\")", + "sqlExpr" : "\"rand(1)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 16, + "fragment" : "rand('1')" + } ] +} diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/group-by-ordinal.sql new file mode 100644 index 000000000000..b773396c050d --- /dev/null +++ b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/group-by-ordinal.sql @@ -0,0 +1,96 @@ +-- group by ordinal positions + +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b); + +-- basic case +select a, sum(b) from data group by 1; + +-- constant case +select 1, 2, sum(b) from data group by 1, 2; + +-- duplicate group by column +select a, 1, sum(b) from data group by a, 1; +select a, 1, sum(b) from data group by 1, 2; + +-- group by a non-aggregate expression's ordinal +select a, b + 2, count(2) from data group by a, 2; + +-- with alias +select a as aa, b + 2 as bb, count(2) from data group by 1, 2; + +-- foldable non-literal: this should be the same as no grouping. +select sum(b) from data group by 1 + 0; + +-- negative cases: ordinal out of range +select a, b from data group by -1; +select a, b from data group by 0; +select a, b from data group by 3; + +-- negative case: position is an aggregate expression +select a, b, sum(b) from data group by 3; +select a, b, sum(b) + 2 from data group by 3; + +-- negative case: nondeterministic expression +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2; + +-- negative case: star +select * from data group by a, b, 1; + +-- group by ordinal followed by order by +select a, count(a) from (select 1 as a) tmp group by 1 order by 1; + +-- group by ordinal followed by having +select count(a), a from (select 1 as a) tmp group by 2 having a > 0; + +-- mixed cases: group-by ordinals and aliases +select a, a AS k, count(b) from data group by k, 1; + +-- can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, 2); + +-- mixed cases: can use ordinal in CUBE +select a, b, count(1) from data group by cube(1, b); + +-- can use ordinal with cube +select a, b, count(1) from data group by 1, 2 with cube; + +-- can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, 2); + +-- mixed cases: can use ordinal in ROLLUP +select a, b, count(1) from data group by rollup(1, b); + +-- can use ordinal with rollup +select a, b, count(1) from data group by 1, 2 with rollup; + +-- can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)); + +-- mixed cases: can use ordinal in GROUPING SETS +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)); + +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)); + +-- range error +select a, b, count(1) from data group by a, -1; + +select a, b, count(1) from data group by a, 3; + +select a, b, count(1) from data group by cube(-1, 2); + +select a, b, count(1) from data group by cube(1, 3); + +-- turn off group by ordinal +set spark.sql.groupByOrdinal=false; + +-- can now group by negative literal +select sum(b) from data group by -1; diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/inputs/random.sql b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/random.sql new file mode 100644 index 000000000000..a1aae7b8759d --- /dev/null +++ b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/random.sql @@ -0,0 +1,17 @@ +-- rand with the seed 0 +SELECT rand(0); +SELECT rand(cast(3 / 7 AS int)); +SELECT rand(NULL); +SELECT rand(cast(NULL AS int)); + +-- rand unsupported data type +SELECT rand(1.0); + +-- randn with the seed 0 +SELECT randn(0L); +SELECT randn(cast(3 / 7 AS long)); +SELECT randn(NULL); +SELECT randn(cast(NULL AS long)); + +-- randn unsupported data type +SELECT rand('1') diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/inputs/table-valued-functions.sql b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/table-valued-functions.sql new file mode 100644 index 000000000000..79d427bc2099 --- /dev/null +++ b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/table-valued-functions.sql @@ -0,0 +1,126 @@ +-- unresolved function +select * from dummy(3); + +-- range call with end +select * from range(6 + cos(3)); + +-- range call with start and end +select * from range(5, 10); + +-- range call with step +select * from range(0, 10, 2); + +-- range call with numPartitions +select * from range(0, 10, 1, 200); + +-- range call with invalid number of arguments +select * from range(1, 1, 1, 1, 1); + +-- range call with null +select * from range(1, null); + +-- range call with incompatible type +select * from range(array(1, 2, 3)); + +-- range call with illegal step +select * from range(0, 5, 0); + +-- range call with a mixed-case function name +select * from RaNgE(2); + +-- range call with alias +select i from range(0, 2) t(i); + +-- explode +select * from explode(array(1, 2)); +select * from explode(map('a', 1, 'b', 2)); + +-- explode with empty values +select * from explode(array()); +select * from explode(map()); + +-- explode with column aliases +select * from explode(array(1, 2)) t(c1); +select * from explode(map('a', 1, 'b', 2)) t(k, v); + +-- explode with non-deterministic values +select * from explode(array(rand(0))); + +-- explode with erroneous input +select * from explode(null); +select * from explode(null) t(c1); +select * from explode(1); +select * from explode(1, 2); +select * from explode(explode(array(1))); +select * from explode(array(1, 2)) t(c1, c2); + +-- explode_outer +select * from explode_outer(array(1, 2)); +select * from explode_outer(map('a', 1, 'b', 2)); +select * from explode_outer(array()); +select * from explode_outer(map()); + +-- table-valued functions with join +select * from range(2) join explode(array(1, 2)); +select * from range(2) join explode_outer(array()); + +-- inline +select * from inline(array(struct(1, 'a'), struct(2, 'b'))); +select * from inline(array(struct(1, 'a'), struct(2, 'b'))) t(x, y); +select * from inline(array_remove(array(struct(1, 'a')), struct(1, 'a'))); + +-- inline with erroneous input +select * from inline(null); +select * from inline(array(struct(1, 2), struct(2, 3))) t(a, b, c); + +-- inline_outer +select * from inline_outer(array(struct(1, 'a'), struct(2, 'b'))); +select * from inline_outer(array_remove(array(struct(1, 'a')), struct(1, 'a'))); + +-- posexplode +select * from posexplode(array()); +select * from posexplode(array(1, 2)); +select * from posexplode(array(1, 2)) t(pos, x); +select * from posexplode(map()); +select * from posexplode(map('a', 1, 'b', 2)); +select * from posexplode(map('a', 1, 'b', 2)) t(pos, k, v); + +-- posexplode with erroneous input +select * from posexplode(1); +select * from posexplode(1, 2); +select * from posexplode(explode(array(1))); +select * from posexplode(array(1, 2)) t(x); + +-- posexplode +select * from posexplode_outer(array()); +select * from posexplode_outer(array(1, 2)); +select * from posexplode_outer(map()); +select * from posexplode_outer(map('a', 1, 'b', 2)); + +-- json_tuple +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'b'); +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'c'); +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'a'); +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'b') AS t(x, y); +select * from json_tuple('{"a": bad, "b": string}', 'a', 'b'); + +-- json_tuple with erroneous input +select * from json_tuple(); +select * from json_tuple('{"a": 1}'); +select * from json_tuple('{"a": 1}', 1); +select * from json_tuple('{"a": 1}', null); +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'b') AS t(x); + +-- stack +select * from stack(1, 1, 2, 3); +select * from stack(2, 1, 2, 3); +select * from stack(3, 1, 2, 3) t(x); +select * from stack(4, 1, 2, 3) t(x); +select * from stack(2, 1, 1.1, 'a', 2, 2.2, 'b') t(a, b, c); +select * from stack(2, 1, 1.1, null, 2, null, 'b') t(a, b, c); + +-- stack with erroneous input +select * from stack(); +select * from stack(2, 1, 2, 3) t(a, b, c); +select * from stack(2, 1, '1.1', 'a', 2, 2.2, 'b'); +select * from stack(2, explode(array(1, 2, 3))); diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by-ordinal.sql.out b/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by-ordinal.sql.out new file mode 100644 index 000000000000..b968b4e09fac --- /dev/null +++ b/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by-ordinal.sql.out @@ -0,0 +1,524 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +create temporary view data as select * from values + (1, 1), + (1, 2), + (2, 1), + (2, 2), + (3, 1), + (3, 2) + as data(a, b) +-- !query schema +struct<> +-- !query output + + + +-- !query +select a, sum(b) from data group by 1 +-- !query schema +struct +-- !query output +1 3 +2 3 +3 3 + + +-- !query +select 1, 2, sum(b) from data group by 1, 2 +-- !query schema +struct<1:int,2:int,sum(b):bigint> +-- !query output +1 2 9 + + +-- !query +select a, 1, sum(b) from data group by a, 1 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, 1, sum(b) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 1 3 +2 1 3 +3 1 3 + + +-- !query +select a, b + 2, count(2) from data group by a, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select a as aa, b + 2 as bb, count(2) from data group by 1, 2 +-- !query schema +struct +-- !query output +1 3 1 +1 4 1 +2 3 1 +2 4 1 +3 3 1 +3 4 1 + + +-- !query +select sum(b) from data group by 1 + 0 +-- !query schema +struct +-- !query output +9 + + +-- !query +select a, b from data group by -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "-1", + "size" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 32, + "stopIndex" : 33, + "fragment" : "-1" + } ] +} + + +-- !query +select a, b from data group by 0 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "0", + "size" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 32, + "stopIndex" : 32, + "fragment" : "0" + } ] +} + + +-- !query +select a, b from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "3", + "size" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 32, + "stopIndex" : 32, + "fragment" : "3" + } ] +} + + +-- !query +select a, b, sum(b) from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "sum(data.b) AS `sum(b)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 40, + "stopIndex" : 40, + "fragment" : "3" + } ] +} + + +-- !query +select a, b, sum(b) + 2 from data group by 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "(sum(data.b) + CAST(2 AS BIGINT)) AS `(sum(b) + 2)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 44, + "stopIndex" : 44, + "fragment" : "3" + } ] +} + + +-- !query +select a, rand(0), sum(b) +from +(select /*+ REPARTITION(1) */ a, b from data) group by a, 2 +-- !query schema +struct +-- !query output +1 0.5488135024422883 1 +1 0.7151893651681639 2 +2 0.5448831775801376 2 +2 0.6027633705776989 1 +3 0.4236547969336536 1 +3 0.6458941151817286 2 + + +-- !query +select * from data group by a, b, 1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "STAR_GROUP_BY_POS", + "sqlState" : "0A000" +} + + +-- !query +select a, count(a) from (select 1 as a) tmp group by 1 order by 1 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select count(a), a from (select 1 as a) tmp group by 2 having a > 0 +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select a, a AS k, count(b) from data group by k, 1 +-- !query schema +struct +-- !query output +1 1 2 +2 2 2 +3 3 2 + + +-- !query +select a, b, count(1) from data group by cube(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by cube(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with cube +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, 2) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by rollup(1, b) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by 1, 2 with rollup +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL NULL 6 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (2), (1, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, 2 grouping sets((1), (b), (a, 2)) +-- !query schema +struct +-- !query output +1 1 1 +1 2 1 +1 NULL 2 +2 1 1 +2 2 1 +2 NULL 2 +3 1 1 +3 2 1 +3 NULL 2 +NULL 1 3 +NULL 2 3 + + +-- !query +select a, b, count(1) from data group by a, -1 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "-1", + "size" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 45, + "stopIndex" : 46, + "fragment" : "-1" + } ] +} + + +-- !query +select a, b, count(1) from data group by a, 3 +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "count(1) AS `count(1)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 45, + "stopIndex" : 45, + "fragment" : "3" + } ] +} + + +-- !query +select a, b, count(1) from data group by cube(-1, 2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_OUT_OF_RANGE", + "sqlState" : "42805", + "messageParameters" : { + "index" : "-1", + "size" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 47, + "stopIndex" : 48, + "fragment" : "-1" + } ] +} + + +-- !query +select a, b, count(1) from data group by cube(1, 3) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "GROUP_BY_POS_AGGREGATE", + "sqlState" : "42903", + "messageParameters" : { + "aggExpr" : "count(1) AS `count(1)`", + "index" : "3" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 50, + "stopIndex" : 50, + "fragment" : "3" + } ] +} + + +-- !query +set spark.sql.groupByOrdinal=false +-- !query schema +struct +-- !query output +spark.sql.groupByOrdinal false + + +-- !query +select sum(b) from data group by -1 +-- !query schema +struct +-- !query output +9 diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by.sql.out b/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by.sql.out index f56420926050..a4a3f76fa6a7 100644 --- a/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by.sql.out +++ b/gluten-ut/spark35/src/test/resources/sql-tests/results/group-by.sql.out @@ -907,7 +907,7 @@ GROUP BY a IS NULL -- !query schema struct<(IF((NOT (a IS NULL)), rand(0), 1)):double,c:bigint> -- !query output -0.7604953758285915 7 +0.5488135024422883 7 1.0 2 diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/results/random.sql.out b/gluten-ut/spark35/src/test/resources/sql-tests/results/random.sql.out new file mode 100644 index 000000000000..17e6f871b9c5 --- /dev/null +++ b/gluten-ut/spark35/src/test/resources/sql-tests/results/random.sql.out @@ -0,0 +1,115 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT rand(0) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(3 / 7 AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(NULL) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(cast(NULL AS int)) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +SELECT rand(1.0) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"1.0\"", + "inputType" : "\"DECIMAL(2,1)\"", + "paramIndex" : "1", + "requiredType" : "(\"INT\" or \"BIGINT\")", + "sqlExpr" : "\"rand(1.0)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 16, + "fragment" : "rand(1.0)" + } ] +} + + +-- !query +SELECT randn(0L) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(3 / 7 AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(NULL) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT randn(cast(NULL AS long)) +-- !query schema +struct +-- !query output +1.6034991609278433 + + +-- !query +SELECT rand('1') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"1\"", + "inputType" : "\"STRING\"", + "paramIndex" : "1", + "requiredType" : "(\"INT\" or \"BIGINT\")", + "sqlExpr" : "\"rand(1)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 16, + "fragment" : "rand('1')" + } ] +} diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/results/table-valued-functions.sql.out b/gluten-ut/spark35/src/test/resources/sql-tests/results/table-valued-functions.sql.out new file mode 100644 index 000000000000..0d5675fa6fde --- /dev/null +++ b/gluten-ut/spark35/src/test/resources/sql-tests/results/table-valued-functions.sql.out @@ -0,0 +1,1017 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +select * from dummy(3) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "UNRESOLVABLE_TABLE_VALUED_FUNCTION", + "messageParameters" : { + "name" : "`dummy`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 22, + "fragment" : "dummy(3)" + } ] +} + + +-- !query +select * from range(6 + cos(3)) +-- !query schema +struct +-- !query output +0 +1 +2 +3 +4 + + +-- !query +select * from range(5, 10) +-- !query schema +struct +-- !query output +5 +6 +7 +8 +9 + + +-- !query +select * from range(0, 10, 2) +-- !query schema +struct +-- !query output +0 +2 +4 +6 +8 + + +-- !query +select * from range(0, 10, 1, 200) +-- !query schema +struct +-- !query output +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + + +-- !query +select * from range(1, 1, 1, 1, 1) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION", + "sqlState" : "42605", + "messageParameters" : { + "actualNum" : "5", + "docroot" : "https://spark.apache.org/docs/latest", + "expectedNum" : "[1, 2, 3, 4]", + "functionName" : "`range`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 34, + "fragment" : "range(1, 1, 1, 1, 1)" + } ] +} + + +-- !query +select * from range(1, null) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "functionName" : "`range`", + "inputSql" : "\"NULL\"", + "inputType" : "\"VOID\"", + "paramIndex" : "2", + "requiredType" : "\"BIGINT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 28, + "fragment" : "range(1, null)" + } ] +} + + +-- !query +select * from range(array(1, 2, 3)) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "functionName" : "`range`", + "inputSql" : "\"array(1, 2, 3)\"", + "inputType" : "\"ARRAY\"", + "paramIndex" : "2", + "requiredType" : "\"BIGINT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 35, + "fragment" : "range(array(1, 2, 3))" + } ] +} + + +-- !query +select * from range(0, 5, 0) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "FAILED_FUNCTION_CALL", + "sqlState" : "38000", + "messageParameters" : { + "funcName" : "`range`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 28, + "fragment" : "range(0, 5, 0)" + } ] +} + + +-- !query +select * from RaNgE(2) +-- !query schema +struct +-- !query output +0 +1 + + +-- !query +select i from range(0, 2) t(i) +-- !query schema +struct +-- !query output +0 +1 + + +-- !query +select * from explode(array(1, 2)) +-- !query schema +struct +-- !query output +1 +2 + + +-- !query +select * from explode(map('a', 1, 'b', 2)) +-- !query schema +struct +-- !query output +a 1 +b 2 + + +-- !query +select * from explode(array()) +-- !query schema +struct +-- !query output + + + +-- !query +select * from explode(map()) +-- !query schema +struct +-- !query output + + + +-- !query +select * from explode(array(1, 2)) t(c1) +-- !query schema +struct +-- !query output +1 +2 + + +-- !query +select * from explode(map('a', 1, 'b', 2)) t(k, v) +-- !query schema +struct +-- !query output +a 1 +b 2 + + +-- !query +select * from explode(array(rand(0))) +-- !query schema +struct +-- !query output +0.5488135024422883 + + +-- !query +select * from explode(null) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"NULL\"", + "inputType" : "\"VOID\"", + "paramIndex" : "1", + "requiredType" : "(\"ARRAY\" or \"MAP\")", + "sqlExpr" : "\"explode(NULL)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 27, + "fragment" : "explode(null)" + } ] +} + + +-- !query +select * from explode(null) t(c1) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"NULL\"", + "inputType" : "\"VOID\"", + "paramIndex" : "1", + "requiredType" : "(\"ARRAY\" or \"MAP\")", + "sqlExpr" : "\"explode(NULL)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 33, + "fragment" : "explode(null) t(c1)" + } ] +} + + +-- !query +select * from explode(1) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"1\"", + "inputType" : "\"INT\"", + "paramIndex" : "1", + "requiredType" : "(\"ARRAY\" or \"MAP\")", + "sqlExpr" : "\"explode(1)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 24, + "fragment" : "explode(1)" + } ] +} + + +-- !query +select * from explode(1, 2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION", + "sqlState" : "42605", + "messageParameters" : { + "actualNum" : "2", + "docroot" : "https://spark.apache.org/docs/latest", + "expectedNum" : "1", + "functionName" : "`explode`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 27, + "fragment" : "explode(1, 2)" + } ] +} + + +-- !query +select * from explode(explode(array(1))) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "UNSUPPORTED_GENERATOR.NESTED_IN_EXPRESSIONS", + "sqlState" : "0A000", + "messageParameters" : { + "expression" : "\"explode(explode(array(1)))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 40, + "fragment" : "explode(explode(array(1)))" + } ] +} + + +-- !query +select * from explode(array(1, 2)) t(c1, c2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "NUM_TABLE_VALUE_ALIASES_MISMATCH", + "messageParameters" : { + "aliasesNum" : "2", + "funcName" : "`explode`", + "outColsNum" : "1" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 44, + "fragment" : "explode(array(1, 2)) t(c1, c2)" + } ] +} + + +-- !query +select * from explode_outer(array(1, 2)) +-- !query schema +struct +-- !query output +1 +2 + + +-- !query +select * from explode_outer(map('a', 1, 'b', 2)) +-- !query schema +struct +-- !query output +a 1 +b 2 + + +-- !query +select * from explode_outer(array()) +-- !query schema +struct +-- !query output +NULL + + +-- !query +select * from explode_outer(map()) +-- !query schema +struct +-- !query output +NULL NULL + + +-- !query +select * from range(2) join explode(array(1, 2)) +-- !query schema +struct +-- !query output +0 1 +0 2 +1 1 +1 2 + + +-- !query +select * from range(2) join explode_outer(array()) +-- !query schema +struct +-- !query output +0 NULL +1 NULL + + +-- !query +select * from inline(array(struct(1, 'a'), struct(2, 'b'))) +-- !query schema +struct +-- !query output +1 a +2 b + + +-- !query +select * from inline(array(struct(1, 'a'), struct(2, 'b'))) t(x, y) +-- !query schema +struct +-- !query output +1 a +2 b + + +-- !query +select * from inline(array_remove(array(struct(1, 'a')), struct(1, 'a'))) +-- !query schema +struct +-- !query output + + + +-- !query +select * from inline(null) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"NULL\"", + "inputType" : "\"VOID\"", + "paramIndex" : "1", + "requiredType" : "\"ARRAY\"", + "sqlExpr" : "\"inline(NULL)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 26, + "fragment" : "inline(null)" + } ] +} + + +-- !query +select * from inline(array(struct(1, 2), struct(2, 3))) t(a, b, c) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "NUM_TABLE_VALUE_ALIASES_MISMATCH", + "messageParameters" : { + "aliasesNum" : "3", + "funcName" : "`inline`", + "outColsNum" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 66, + "fragment" : "inline(array(struct(1, 2), struct(2, 3))) t(a, b, c)" + } ] +} + + +-- !query +select * from inline_outer(array(struct(1, 'a'), struct(2, 'b'))) +-- !query schema +struct +-- !query output +1 a +2 b + + +-- !query +select * from inline_outer(array_remove(array(struct(1, 'a')), struct(1, 'a'))) +-- !query schema +struct +-- !query output +NULL NULL + + +-- !query +select * from posexplode(array()) +-- !query schema +struct +-- !query output + + + +-- !query +select * from posexplode(array(1, 2)) +-- !query schema +struct +-- !query output +0 1 +1 2 + + +-- !query +select * from posexplode(array(1, 2)) t(pos, x) +-- !query schema +struct +-- !query output +0 1 +1 2 + + +-- !query +select * from posexplode(map()) +-- !query schema +struct +-- !query output + + + +-- !query +select * from posexplode(map('a', 1, 'b', 2)) +-- !query schema +struct +-- !query output +0 a 1 +1 b 2 + + +-- !query +select * from posexplode(map('a', 1, 'b', 2)) t(pos, k, v) +-- !query schema +struct +-- !query output +0 a 1 +1 b 2 + + +-- !query +select * from posexplode(1) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"1\"", + "inputType" : "\"INT\"", + "paramIndex" : "1", + "requiredType" : "(\"ARRAY\" or \"MAP\")", + "sqlExpr" : "\"posexplode(1)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 27, + "fragment" : "posexplode(1)" + } ] +} + + +-- !query +select * from posexplode(1, 2) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION", + "sqlState" : "42605", + "messageParameters" : { + "actualNum" : "2", + "docroot" : "https://spark.apache.org/docs/latest", + "expectedNum" : "1", + "functionName" : "`posexplode`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 30, + "fragment" : "posexplode(1, 2)" + } ] +} + + +-- !query +select * from posexplode(explode(array(1))) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "UNSUPPORTED_GENERATOR.NESTED_IN_EXPRESSIONS", + "sqlState" : "0A000", + "messageParameters" : { + "expression" : "\"posexplode(explode(array(1)))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 43, + "fragment" : "posexplode(explode(array(1)))" + } ] +} + + +-- !query +select * from posexplode(array(1, 2)) t(x) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "NUM_TABLE_VALUE_ALIASES_MISMATCH", + "messageParameters" : { + "aliasesNum" : "1", + "funcName" : "`posexplode`", + "outColsNum" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 42, + "fragment" : "posexplode(array(1, 2)) t(x)" + } ] +} + + +-- !query +select * from posexplode_outer(array()) +-- !query schema +struct +-- !query output +NULL NULL + + +-- !query +select * from posexplode_outer(array(1, 2)) +-- !query schema +struct +-- !query output +0 1 +1 2 + + +-- !query +select * from posexplode_outer(map()) +-- !query schema +struct +-- !query output +NULL NULL NULL + + +-- !query +select * from posexplode_outer(map('a', 1, 'b', 2)) +-- !query schema +struct +-- !query output +0 a 1 +1 b 2 + + +-- !query +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'b') +-- !query schema +struct +-- !query output +1 2 + + +-- !query +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'c') +-- !query schema +struct +-- !query output +1 NULL + + +-- !query +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'a') +-- !query schema +struct +-- !query output +1 1 + + +-- !query +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'b') AS t(x, y) +-- !query schema +struct +-- !query output +1 2 + + +-- !query +select * from json_tuple('{"a": bad, "b": string}', 'a', 'b') +-- !query schema +struct +-- !query output +NULL NULL + + +-- !query +select * from json_tuple() +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION", + "sqlState" : "42605", + "messageParameters" : { + "actualNum" : "0", + "docroot" : "https://spark.apache.org/docs/latest", + "expectedNum" : "> 1", + "functionName" : "`json_tuple`" + } +} + + +-- !query +select * from json_tuple('{"a": 1}') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION", + "sqlState" : "42605", + "messageParameters" : { + "actualNum" : "1", + "docroot" : "https://spark.apache.org/docs/latest", + "expectedNum" : "> 1", + "functionName" : "`json_tuple`" + } +} + + +-- !query +select * from json_tuple('{"a": 1}', 1) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.NON_STRING_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "funcName" : "`json_tuple`", + "sqlExpr" : "\"json_tuple({\"a\": 1}, 1)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 39, + "fragment" : "json_tuple('{\"a\": 1}', 1)" + } ] +} + + +-- !query +select * from json_tuple('{"a": 1}', null) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.NON_STRING_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "funcName" : "`json_tuple`", + "sqlExpr" : "\"json_tuple({\"a\": 1}, NULL)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 42, + "fragment" : "json_tuple('{\"a\": 1}', null)" + } ] +} + + +-- !query +select * from json_tuple('{"a": 1, "b": 2}', 'a', 'b') AS t(x) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "NUM_TABLE_VALUE_ALIASES_MISMATCH", + "messageParameters" : { + "aliasesNum" : "1", + "funcName" : "`json_tuple`", + "outColsNum" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 62, + "fragment" : "json_tuple('{\"a\": 1, \"b\": 2}', 'a', 'b') AS t(x)" + } ] +} + + +-- !query +select * from stack(1, 1, 2, 3) +-- !query schema +struct +-- !query output +1 2 3 + + +-- !query +select * from stack(2, 1, 2, 3) +-- !query schema +struct +-- !query output +1 2 +3 NULL + + +-- !query +select * from stack(3, 1, 2, 3) t(x) +-- !query schema +struct +-- !query output +1 +2 +3 + + +-- !query +select * from stack(4, 1, 2, 3) t(x) +-- !query schema +struct +-- !query output +1 +2 +3 +NULL + + +-- !query +select * from stack(2, 1, 1.1, 'a', 2, 2.2, 'b') t(a, b, c) +-- !query schema +struct +-- !query output +1 1.1 a +2 2.2 b + + +-- !query +select * from stack(2, 1, 1.1, null, 2, null, 'b') t(a, b, c) +-- !query schema +struct +-- !query output +1 1.1 NULL +2 NULL b + + +-- !query +select * from stack() +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "WRONG_NUM_ARGS.WITHOUT_SUGGESTION", + "sqlState" : "42605", + "messageParameters" : { + "actualNum" : "0", + "docroot" : "https://spark.apache.org/docs/latest", + "expectedNum" : "> 1", + "functionName" : "`stack`" + } +} + + +-- !query +select * from stack(2, 1, 2, 3) t(a, b, c) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "NUM_TABLE_VALUE_ALIASES_MISMATCH", + "messageParameters" : { + "aliasesNum" : "3", + "funcName" : "`stack`", + "outColsNum" : "2" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 42, + "fragment" : "stack(2, 1, 2, 3) t(a, b, c)" + } ] +} + + +-- !query +select * from stack(2, 1, '1.1', 'a', 2, 2.2, 'b') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.STACK_COLUMN_DIFF_TYPES", + "sqlState" : "42K09", + "messageParameters" : { + "columnIndex" : "1", + "leftParamIndex" : "2", + "leftType" : "\"STRING\"", + "rightParamIndex" : "5", + "rightType" : "\"DECIMAL(2,1)\"", + "sqlExpr" : "\"stack(2, 1, 1.1, a, 2, 2.2, b)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 50, + "fragment" : "stack(2, 1, '1.1', 'a', 2, 2.2, 'b')" + } ] +} + + +-- !query +select * from stack(2, explode(array(1, 2, 3))) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "UNSUPPORTED_GENERATOR.NESTED_IN_EXPRESSIONS", + "sqlState" : "0A000", + "messageParameters" : { + "expression" : "\"stack(2, explode(array(1, 2, 3)))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 15, + "stopIndex" : 47, + "fragment" : "stack(2, explode(array(1, 2, 3)))" + } ] +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 40ecc3c351e6..76b66677963f 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1132,6 +1132,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-41017: filter pushdown with nondeterministic predicates") enableSuite[GlutenFileScanSuite] enableSuite[GlutenGeneratorFunctionSuite] + .exclude("SPARK-45171: Handle evaluated nondeterministic expression") enableSuite[GlutenInjectRuntimeFilterSuite] // FIXME: yan .exclude("Merge runtime bloom filters") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala index 2421e918bf21..b3d51e802985 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenGeneratorFunctionSuite.scala @@ -16,4 +16,13 @@ */ package org.apache.spark.sql -class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.internal.SQLConf + +class GlutenGeneratorFunctionSuite extends GeneratorFunctionSuite with GlutenSQLTestsTrait { + testGluten("SPARK-45171: Handle evaluated nondeterministic expression") { + withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") { + val df = sql("select explode(array(rand(0)))") + checkAnswer(df, Row(0.5488135024422883)) + } + } +} From 9a38eba11aaab68d7e0722a1029fe7412acbbfb1 Mon Sep 17 00:00:00 2001 From: Rui Mo Date: Fri, 24 May 2024 08:45:46 +0800 Subject: [PATCH 140/402] [VL] Enable NaN tests for array functions (#5854) --- .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 ------ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 ------ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 ------ .../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 ------ 4 files changed, 24 deletions(-) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5e35912034b8..c78d8230e3a6 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -202,12 +202,6 @@ class VeloxTestSettings extends BackendTestSettings { // Rewrite in Gluten to replace Seq with Array .exclude("Shuffle") .excludeGlutenTest("Shuffle") - // TODO: ArrayDistinct should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36741") - // TODO: ArrayIntersect should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36754") - // Not supported case. - .exclude("SPARK-36753: ArrayExcept should handle duplicated Double.NaN and Float.Nan") enableSuite[GlutenDateExpressionsSuite] // Rewrite because Spark collect causes long overflow. .exclude("TIMESTAMP_MICROS") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 1d796aa1b74a..3b32cebca13f 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -115,12 +115,6 @@ class VeloxTestSettings extends BackendTestSettings { // Rewrite in Gluten to replace Seq with Array .exclude("Shuffle") .excludeGlutenTest("Shuffle") - // TODO: ArrayDistinct should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36741") - // TODO: ArrayIntersect should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36754") - // Not supported case. - .exclude("SPARK-36753: ArrayExcept should handle duplicated Double.NaN and Float.Nan") enableSuite[GlutenConditionalExpressionSuite] enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 7c8509f8034d..3a993189d6ae 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -96,12 +96,6 @@ class VeloxTestSettings extends BackendTestSettings { // Rewrite in Gluten to replace Seq with Array .exclude("Shuffle") .excludeGlutenTest("Shuffle") - // TODO: ArrayDistinct should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36741") - // TODO: ArrayIntersect should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36754") - // Not supported case. - .exclude("SPARK-36753: ArrayExcept should handle duplicated Double.NaN and Float.Nan") enableSuite[GlutenConditionalExpressionSuite] enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 76b66677963f..98942462aec8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -97,12 +97,6 @@ class VeloxTestSettings extends BackendTestSettings { // Rewrite in Gluten to replace Seq with Array .exclude("Shuffle") .excludeGlutenTest("Shuffle") - // TODO: ArrayDistinct should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36741") - // TODO: ArrayIntersect should handle duplicated Double.NaN - .excludeByPrefix("SPARK-36754") - // Not supported case. - .exclude("SPARK-36753: ArrayExcept should handle duplicated Double.NaN and Float.Nan") enableSuite[GlutenConditionalExpressionSuite] enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. From ff84bcd8f2e4bbbe234c6c36985d2705b267a8d1 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 24 May 2024 09:19:55 +0800 Subject: [PATCH 141/402] [CORE] Remove static modifier on TreeMemoryConsumers.Factory.map (#5849) --- dev/ci-velox-buildstatic.sh | 2 +- .../memtarget/spark/TreeMemoryConsumers.java | 10 +- .../spark/TreeMemoryConsumerTest.java | 122 ++++++++++++++++++ 3 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic.sh index 208490d1c2eb..0754408169cf 100755 --- a/dev/ci-velox-buildstatic.sh +++ b/dev/ci-velox-buildstatic.sh @@ -2,7 +2,7 @@ yum install sudo patch java-1.8.0-openjdk-devel -y cd $GITHUB_WORKSPACE/ep/build-velox/src ./get_velox.sh source /opt/rh/devtoolset-9/enable -source /opt/gluten/dev/vcpkg/env.sh +source $GITHUB_WORKSPACE/dev/vcpkg/env.sh cd $GITHUB_WORKSPACE/ sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt export NUM_THREADS=4 diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java index 46257d80e7dd..1da23d15e353 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java @@ -30,7 +30,6 @@ import java.util.concurrent.ConcurrentHashMap; public final class TreeMemoryConsumers { - private static final Map FACTORIES = new ConcurrentHashMap<>(); private TreeMemoryConsumers() {} @@ -61,8 +60,7 @@ public static Factory shared() { } public static class Factory { - - private static final ReferenceMap MAP = new ReferenceMap(ReferenceMap.WEAK, ReferenceMap.WEAK); + private final ReferenceMap map = new ReferenceMap(ReferenceMap.WEAK, ReferenceMap.WEAK); private final long perTaskCapacity; private Factory(long perTaskCapacity) { @@ -71,9 +69,9 @@ private Factory(long perTaskCapacity) { @SuppressWarnings("unchecked") private TreeMemoryTarget getSharedAccount(TaskMemoryManager tmm) { - synchronized (MAP) { + synchronized (map) { return (TreeMemoryTarget) - MAP.computeIfAbsent( + map.computeIfAbsent( tmm, m -> { TreeMemoryTarget tmc = new TreeMemoryConsumer((TaskMemoryManager) m); @@ -88,7 +86,7 @@ public TreeMemoryTarget newConsumer( String name, List spillers, Map virtualChildren) { - TreeMemoryTarget account = getSharedAccount(tmm); + final TreeMemoryTarget account = getSharedAccount(tmm); return account.newChild( name, TreeMemoryConsumer.CAPACITY_UNLIMITED, spillers, virtualChildren); } diff --git a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java new file mode 100644 index 000000000000..e26765d33082 --- /dev/null +++ b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.memory.memtarget.spark; + +import org.apache.gluten.GlutenConfig; +import org.apache.gluten.memory.memtarget.TreeMemoryTarget; + +import org.apache.spark.TaskContext; +import org.apache.spark.sql.internal.SQLConf; +import org.apache.spark.util.TaskResources$; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +import scala.Function0; + +public class TreeMemoryConsumerTest { + @Test + public void testIsolated() { + final SQLConf conf = new SQLConf(); + conf.setConfString( + GlutenConfig.COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES().key(), "100"); + test( + conf, + () -> { + final TreeMemoryConsumers.Factory factory = TreeMemoryConsumers.isolated(); + final TreeMemoryTarget consumer = + factory.newConsumer( + TaskContext.get().taskMemoryManager(), + "FOO", + Collections.emptyList(), + Collections.emptyMap()); + Assert.assertEquals(20, consumer.borrow(20)); + Assert.assertEquals(70, consumer.borrow(70)); + Assert.assertEquals(10, consumer.borrow(20)); + Assert.assertEquals(0, consumer.borrow(20)); + }); + } + + @Test + public void testShared() { + final SQLConf conf = new SQLConf(); + conf.setConfString( + GlutenConfig.COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES().key(), "100"); + test( + conf, + () -> { + final TreeMemoryConsumers.Factory factory = TreeMemoryConsumers.shared(); + final TreeMemoryTarget consumer = + factory.newConsumer( + TaskContext.get().taskMemoryManager(), + "FOO", + Collections.emptyList(), + Collections.emptyMap()); + Assert.assertEquals(20, consumer.borrow(20)); + Assert.assertEquals(70, consumer.borrow(70)); + Assert.assertEquals(20, consumer.borrow(20)); + Assert.assertEquals(20, consumer.borrow(20)); + }); + } + + @Test + public void testIsolatedAndShared() { + final SQLConf conf = new SQLConf(); + conf.setConfString( + GlutenConfig.COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES().key(), "100"); + test( + conf, + () -> { + final TreeMemoryTarget shared = + TreeMemoryConsumers.shared() + .newConsumer( + TaskContext.get().taskMemoryManager(), + "FOO", + Collections.emptyList(), + Collections.emptyMap()); + Assert.assertEquals(110, shared.borrow(110)); + final TreeMemoryTarget isolated = + TreeMemoryConsumers.isolated() + .newConsumer( + TaskContext.get().taskMemoryManager(), + "FOO", + Collections.emptyList(), + Collections.emptyMap()); + Assert.assertEquals(100, isolated.borrow(110)); + }); + } + + private void test(SQLConf conf, Runnable r) { + TaskResources$.MODULE$.runUnsafe( + new Function0() { + @Override + public Object apply() { + SQLConf.withExistingConf( + conf, + new Function0() { + @Override + public Object apply() { + r.run(); + return null; + } + }); + return null; + } + }); + } +} From ec3f6b318ef09d1f697997efdbb24e75bd2e0835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Fri, 24 May 2024 09:27:04 +0800 Subject: [PATCH 142/402] [GLUTEN-4039][VL] Support width_bucket function (#5634) --- .../gluten/execution/ScalarFunctionsValidateSuite.scala | 7 +++++++ .../org/apache/gluten/expression/ExpressionMappings.scala | 1 + .../org/apache/gluten/expression/ExpressionNames.scala | 1 + 3 files changed, 9 insertions(+) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 3180842adb16..f9ec07619281 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -594,6 +594,13 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + testWithSpecifiedSparkVersion("Test width_bucket function", Some("3.4")) { + runQueryAndCompare("""SELECT width_bucket(2, 0, 4, 3), l_orderkey + | from lineitem limit 100""".stripMargin) { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + testWithSpecifiedSparkVersion("Test url_decode function", Some("3.4")) { withTempPath { path => diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index c734967dea68..14371a71ecdc 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -275,6 +275,7 @@ object ExpressionMappings { Sig[PromotePrecision](PROMOTE_PRECISION), Sig[MonotonicallyIncreasingID](MONOTONICALLY_INCREASING_ID), Sig[SparkPartitionID](SPARK_PARTITION_ID), + Sig[WidthBucket](WIDTH_BUCKET), // Decimal Sig[UnscaledValue](UNSCALED_VALUE), // Generator function diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index eded85e06006..6e6502c19b48 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -302,6 +302,7 @@ object ExpressionNames { final val PROMOTE_PRECISION = "promote_precision" final val SPARK_PARTITION_ID = "spark_partition_id" final val MONOTONICALLY_INCREASING_ID = "monotonically_increasing_id" + final val WIDTH_BUCKET = "width_bucket" // Directly use child expression transformer final val KNOWN_NULLABLE = "known_nullable" From aeea6498ece025004d71e1090141a25d39771245 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 24 May 2024 11:40:08 +0800 Subject: [PATCH 143/402] [VL] Daily Update Velox Version (2024_05_24) (#5860) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index af032e186637..6c25c8f08426 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_23 +VELOX_BRANCH=2024_05_24 VELOX_HOME="" #Set on run gluten on HDFS From 891ab83cca70dfec4c4c700cf73febc64fea73c5 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Fri, 24 May 2024 02:23:05 -0500 Subject: [PATCH 144/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240524) (#5857) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240524) * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/63058 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- cpp-ch/local-engine/Storages/IO/NativeReader.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 04bd6960ee83..1bc0454a1f1d 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240522 -CH_COMMIT=327f885e4bd \ No newline at end of file +CH_BRANCH=rebase_ch/20240524 +CH_COMMIT=b1a2a14bdd2 \ No newline at end of file diff --git a/cpp-ch/local-engine/Storages/IO/NativeReader.cpp b/cpp-ch/local-engine/Storages/IO/NativeReader.cpp index 30d8af593421..48e6950e27eb 100644 --- a/cpp-ch/local-engine/Storages/IO/NativeReader.cpp +++ b/cpp-ch/local-engine/Storages/IO/NativeReader.cpp @@ -128,7 +128,7 @@ static void readNormalSimpleData(DB::ReadBuffer &in, DB::ColumnPtr & column, siz ISerialization::DeserializeBinaryBulkStatePtr state; - column_parse_util.serializer->deserializeBinaryBulkStatePrefix(settings, state); + column_parse_util.serializer->deserializeBinaryBulkStatePrefix(settings, state, nullptr); column_parse_util.serializer->deserializeBinaryBulkWithMultipleStreams(column, rows, settings, state, nullptr); } @@ -145,7 +145,7 @@ readNormalComplexData(DB::ReadBuffer & in, DB::ColumnPtr & column, size_t rows, ISerialization::DeserializeBinaryBulkStatePtr state; DB::ColumnPtr new_col = column->cloneResized(0); - column_parse_util.serializer->deserializeBinaryBulkStatePrefix(settings, state); + column_parse_util.serializer->deserializeBinaryBulkStatePrefix(settings, state ,nullptr); column_parse_util.serializer->deserializeBinaryBulkWithMultipleStreams(new_col, rows, settings, state, nullptr); column->assumeMutable()->insertRangeFrom(*new_col, 0, new_col->size()); } From 9f424a1d05180fcbd00ea9b84a8d3d8b7dcca8c0 Mon Sep 17 00:00:00 2001 From: Sandino Flores Date: Fri, 24 May 2024 15:19:34 -0600 Subject: [PATCH 145/402] [VL] Add support for GCS retry properties (#5858) The Velox GCS connector was recently added a couple of properties: HiveConfig::kGCSMaxRetryCount: Retry until a number of transient errors is detected. HiveConfig::kGCSMaxRetryTime: Retry until a time elapses. This properties are useful to allow a client to either fail immediately or keep retring for several minutes. If none of them is set, then it will use the defaults set by GCS. This change was tested manually, by entering an unnaccesible GCS bucket then verifying that the properties are honored. --- cpp/velox/utils/ConfigExtractor.cc | 13 +++++++++++++ docs/get-started/VeloxGCS.md | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index 0cbba37a783d..a71f143225b9 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -121,6 +121,19 @@ std::shared_ptr getHiveConfig(std::shared_ptr< } } + // https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md#http-transport-configuration + // https://cloud.google.com/cpp/docs/reference/storage/latest/classgoogle_1_1cloud_1_1storage_1_1LimitedErrorCountRetryPolicy + auto gsMaxRetryCount = conf->get("spark.hadoop.fs.gs.http.max.retry"); + if (gsMaxRetryCount.hasValue()) { + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSMaxRetryCount] = gsMaxRetryCount.value(); + } + + // https://cloud.google.com/cpp/docs/reference/storage/latest/classgoogle_1_1cloud_1_1storage_1_1LimitedTimeRetryPolicy + auto gsMaxRetryTime = conf->get("spark.hadoop.fs.gs.http.max.retry-time"); + if (gsMaxRetryTime.hasValue()) { + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSMaxRetryTime] = gsMaxRetryTime.value(); + } + // https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md#authentication auto gsAuthType = conf->get("spark.hadoop.fs.gs.auth.type"); if (gsAuthType.hasValue()) { diff --git a/docs/get-started/VeloxGCS.md b/docs/get-started/VeloxGCS.md index d87d9b960c47..453bee117c1e 100644 --- a/docs/get-started/VeloxGCS.md +++ b/docs/get-started/VeloxGCS.md @@ -36,4 +36,18 @@ spark.hadoop.fs.gs.auth.service.account.json.keyfile // path to the json file wi For cases when a GCS mock is used, an optional endpoint can be provided: ```sh spark.hadoop.fs.gs.storage.root.url // url to the mock gcs service including starting with http or https +``` + +## Configuring GCS max retry count + +For cases when a transient server error is detected, GCS can be configured to keep retrying until a number of transient error is detected. +```sh +spark.hadoop.fs.gs.http.max.retry // number of times to keep retrying unless a non-transient error is detected +``` + +## Configuring GCS max retry time + +For cases when a transient server error is detected, GCS can be configured to keep retrying until the retry loop exceeds a prescribed duration. +```sh +spark.hadoop.fs.gs.http.max.retry-time // a string representing the time keep retring (10s, 1m, etc). ``` \ No newline at end of file From 56c5a24c1bc2886e2cae697711bcda016d0c0cca Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Sun, 26 May 2024 05:35:08 -0500 Subject: [PATCH 146/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240526) (#5870) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240526) * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/59767 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- .../Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp | 8 +++++--- .../Disks/ObjectStorages/GlutenHDFSObjectStorage.h | 2 +- .../ObjectStorages/registerGlutenDiskObjectStorage.cpp | 3 +-- .../local-engine/Storages/Output/WriteBufferBuilder.cpp | 6 ++---- .../Storages/SubstraitSource/ExcelTextFormatFile.cpp | 7 ++----- .../Storages/SubstraitSource/ReadBufferBuilder.cpp | 6 +++--- 7 files changed, 16 insertions(+), 20 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 1bc0454a1f1d..775667afb07d 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240524 -CH_COMMIT=b1a2a14bdd2 \ No newline at end of file +CH_BRANCH=rebase_ch/20240526 +CH_COMMIT=ff17e067fac \ No newline at end of file diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp index 3a844a91f804..60b82ec845bb 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp @@ -17,7 +17,7 @@ #include "GlutenHDFSObjectStorage.h" #if USE_HDFS -#include +#include using namespace DB; namespace local_engine { @@ -35,8 +35,10 @@ std::unique_ptr GlutenHDFSObjectStorage::readObject( /// DB::ObjectStorageKey local_engine::GlutenHDFSObjectStorage::generateObjectKeyForPath(const std::string & path) const { - return DB::ObjectStorageKey::createAsAbsolute(hdfs_root_path + path); + initializeHDFSFS(); + /// what ever data_source_description.description value is, consider that key as relative key + chassert(data_directory.starts_with("/")); + return ObjectStorageKey::createAsRelative(fs::path(url_without_path) / data_directory.substr(1) / path); } } #endif - diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h index 1efa441c2142..a532c98cb87d 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h @@ -33,7 +33,7 @@ class GlutenHDFSObjectStorage final : public DB::HDFSObjectStorage const String & hdfs_root_path_, SettingsPtr settings_, const Poco::Util::AbstractConfiguration & config_) - : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_), config(config_) + : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_, /* lazy_initialize */true), config(config_) { } std::unique_ptr readObject( /// NOLINT diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp index 8f20080297c6..800b51f93e94 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp @@ -75,7 +75,7 @@ void registerGlutenS3ObjectStorage(ObjectStorageFactory & factory) auto uri = getS3URI(config, config_prefix, context); auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); auto settings = getSettings(config, config_prefix, context); - auto client = getClient(config, config_prefix, context, *settings); + auto client = getClient(config, config_prefix, context, *settings, true); auto key_generator = createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key); auto object_storage = std::make_shared( @@ -110,7 +110,6 @@ void registerGlutenHDFSObjectStorage(ObjectStorageFactory & factory) std::unique_ptr settings = std::make_unique( config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), - config.getInt(config_prefix + ".objects_chunk_size_to_delete", 1000), context->getSettingsRef().hdfs_replication ); return std::make_unique(uri, std::move(settings), config); diff --git a/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp b/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp index 02f9cd9f290c..3dc205c39cbd 100644 --- a/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp +++ b/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp @@ -17,13 +17,11 @@ #include #include #include -#include #include -#include -#include +#include +#include #include #include -#include #include #include diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp index d7cbcaa5fe94..038f280b0560 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "Common/CHUtil.h" #include "ExcelTextFormatFile.h" - +#include #include #include -#include #include #include @@ -28,9 +26,8 @@ #include #include #include -#include #include -#include +#include #include #include #include diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp index 21640fe490c7..d54ff985ec5f 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp @@ -35,9 +35,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include From a07e3b8f4f649ce77031eba76fe99c5348f673f0 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 27 May 2024 09:53:29 +0800 Subject: [PATCH 147/402] [VL] RAS: Add config option for setting user cost model, remove fallback strategies from RAS rules list (#5861) --- .github/workflows/velox_docker.yml | 4 +- .../v1-bhj-ras/spark32/1.txt | 105 +-- .../v1-bhj-ras/spark32/10.txt | 260 +++---- .../v1-bhj-ras/spark32/11.txt | 223 +++--- .../v1-bhj-ras/spark32/12.txt | 160 +++-- .../v1-bhj-ras/spark32/13.txt | 193 ++--- .../v1-bhj-ras/spark32/14.txt | 130 ++-- .../v1-bhj-ras/spark32/15.txt | 168 +++-- .../v1-bhj-ras/spark32/16.txt | 253 +++---- .../v1-bhj-ras/spark32/18.txt | 333 +++++---- .../v1-bhj-ras/spark32/19.txt | 126 ++-- .../v1-bhj-ras/spark32/20.txt | 434 +++++------ .../v1-bhj-ras/spark32/21.txt | 357 ++++----- .../v1-bhj-ras/spark32/22.txt | 143 ++-- .../v1-bhj-ras/spark32/3.txt | 205 +++--- .../v1-bhj-ras/spark32/4.txt | 168 +++-- .../v1-bhj-ras/spark32/5.txt | 388 +++++----- .../v1-bhj-ras/spark32/6.txt | 71 +- .../v1-bhj-ras/spark32/7.txt | 361 +++++----- .../v1-bhj-ras/spark32/8.txt | 502 +++++++------ .../v1-bhj-ras/spark32/9.txt | 380 +++++----- .../v1-bhj-ras/spark33/1.txt | 105 +-- .../v1-bhj-ras/spark33/10.txt | 260 +++---- .../v1-bhj-ras/spark33/11.txt | 390 +++++----- .../v1-bhj-ras/spark33/12.txt | 160 +++-- .../v1-bhj-ras/spark33/13.txt | 193 ++--- .../v1-bhj-ras/spark33/14.txt | 130 ++-- .../v1-bhj-ras/spark33/15.txt | 257 +++---- .../v1-bhj-ras/spark33/16.txt | 253 +++---- .../v1-bhj-ras/spark33/18.txt | 333 +++++---- .../v1-bhj-ras/spark33/19.txt | 126 ++-- .../v1-bhj-ras/spark33/20.txt | 424 +++++------ .../v1-bhj-ras/spark33/21.txt | 353 ++++----- .../v1-bhj-ras/spark33/22.txt | 216 +++--- .../v1-bhj-ras/spark33/3.txt | 205 +++--- .../v1-bhj-ras/spark33/4.txt | 168 +++-- .../v1-bhj-ras/spark33/5.txt | 388 +++++----- .../v1-bhj-ras/spark33/6.txt | 71 +- .../v1-bhj-ras/spark33/7.txt | 361 +++++----- .../v1-bhj-ras/spark33/8.txt | 502 +++++++------ .../v1-bhj-ras/spark33/9.txt | 380 +++++----- .../v1-bhj-ras/spark34/1.txt | 105 +-- .../v1-bhj-ras/spark34/10.txt | 260 +++---- .../v1-bhj-ras/spark34/11.txt | 390 +++++----- .../v1-bhj-ras/spark34/12.txt | 160 +++-- .../v1-bhj-ras/spark34/13.txt | 193 ++--- .../v1-bhj-ras/spark34/14.txt | 130 ++-- .../v1-bhj-ras/spark34/15.txt | 257 +++---- .../v1-bhj-ras/spark34/16.txt | 254 +++---- .../v1-bhj-ras/spark34/18.txt | 333 +++++---- .../v1-bhj-ras/spark34/19.txt | 126 ++-- .../v1-bhj-ras/spark34/20.txt | 375 +++++----- .../v1-bhj-ras/spark34/21.txt | 353 ++++----- .../v1-bhj-ras/spark34/22.txt | 216 +++--- .../v1-bhj-ras/spark34/3.txt | 205 +++--- .../v1-bhj-ras/spark34/4.txt | 168 +++-- .../v1-bhj-ras/spark34/5.txt | 388 +++++----- .../v1-bhj-ras/spark34/6.txt | 71 +- .../v1-bhj-ras/spark34/7.txt | 361 +++++----- .../v1-bhj-ras/spark34/8.txt | 502 +++++++------ .../v1-bhj-ras/spark34/9.txt | 380 +++++----- .../tpch-approved-plan/v1-ras/spark32/1.txt | 105 +-- .../tpch-approved-plan/v1-ras/spark32/10.txt | 334 +++++---- .../tpch-approved-plan/v1-ras/spark32/11.txt | 275 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 186 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 171 ++--- .../tpch-approved-plan/v1-ras/spark32/14.txt | 134 ++-- .../tpch-approved-plan/v1-ras/spark32/15.txt | 172 ++--- .../tpch-approved-plan/v1-ras/spark32/16.txt | 244 ++++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 375 +++++----- .../tpch-approved-plan/v1-ras/spark32/19.txt | 130 ++-- .../tpch-approved-plan/v1-ras/spark32/20.txt | 485 +++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 457 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 173 ++--- .../tpch-approved-plan/v1-ras/spark32/3.txt | 227 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 514 ++++++------- .../tpch-approved-plan/v1-ras/spark32/6.txt | 71 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 491 +++++++------ .../tpch-approved-plan/v1-ras/spark32/8.txt | 680 +++++++++--------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 510 ++++++------- .../tpch-approved-plan/v1-ras/spark33/1.txt | 105 +-- .../tpch-approved-plan/v1-ras/spark33/10.txt | 334 +++++---- .../tpch-approved-plan/v1-ras/spark33/11.txt | 468 ++++++------ .../tpch-approved-plan/v1-ras/spark33/12.txt | 186 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 171 ++--- .../tpch-approved-plan/v1-ras/spark33/14.txt | 134 ++-- .../tpch-approved-plan/v1-ras/spark33/15.txt | 261 +++---- .../tpch-approved-plan/v1-ras/spark33/16.txt | 244 ++++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 375 +++++----- .../tpch-approved-plan/v1-ras/spark33/19.txt | 130 ++-- .../tpch-approved-plan/v1-ras/spark33/20.txt | 475 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 453 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 246 ++++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 227 +++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 514 ++++++------- .../tpch-approved-plan/v1-ras/spark33/6.txt | 71 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 491 +++++++------ .../tpch-approved-plan/v1-ras/spark33/8.txt | 680 +++++++++--------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 510 ++++++------- .../tpch-approved-plan/v1-ras/spark34/1.txt | 105 +-- .../tpch-approved-plan/v1-ras/spark34/10.txt | 334 +++++---- .../tpch-approved-plan/v1-ras/spark34/11.txt | 468 ++++++------ .../tpch-approved-plan/v1-ras/spark34/12.txt | 186 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 171 ++--- .../tpch-approved-plan/v1-ras/spark34/14.txt | 134 ++-- .../tpch-approved-plan/v1-ras/spark34/15.txt | 261 +++---- .../tpch-approved-plan/v1-ras/spark34/16.txt | 244 ++++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 375 +++++----- .../tpch-approved-plan/v1-ras/spark34/19.txt | 130 ++-- .../tpch-approved-plan/v1-ras/spark34/20.txt | 475 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 453 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 246 ++++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 227 +++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 514 ++++++------- .../tpch-approved-plan/v1-ras/spark34/6.txt | 71 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 491 +++++++------ .../tpch-approved-plan/v1-ras/spark34/8.txt | 680 +++++++++--------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 510 ++++++------- .../gluten/execution/VeloxTPCHSuite.scala | 4 +- .../apache/gluten/planner/VeloxRasSuite.scala | 67 +- docs/Configuration.md | 2 +- .../enumerated/EnumeratedApplier.scala | 49 +- .../enumerated/EnumeratedTransform.scala | 9 +- .../columnar/enumerated/RemoveFilter.scala | 37 +- .../columnar/heuristic/HeuristicApplier.scala | 4 +- .../columnar/util/AdaptiveContext.scala | 9 +- .../gluten/planner/GlutenOptimization.scala | 45 +- .../gluten/planner/cost/GlutenCostModel.scala | 32 +- .../spark/sql/utils/ReflectionUtil.scala | 28 + .../org/apache/gluten/GlutenConfig.scala | 36 +- 136 files changed, 18469 insertions(+), 16436 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 9b00535b474d..5c39fec35a78 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -189,10 +189,10 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.sql.ras.enabled=true \ + --extra-conf=spark.gluten.ras.enabled=true \ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.sql.ras.enabled=true + --extra-conf=spark.gluten.ras.enabled=true run-tpc-test-ubuntu-oom: needs: build-native-lib-centos-7 diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 699c17ea4562..8d67aad16c3d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index 5b7e141a4beb..c0153da0cda9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,64 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ FlushableHashAggregateExecTransformer (28) - +- ^ ProjectExecTransformer (27) - +- ^ BroadcastHashJoinExecTransformer Inner (26) - :- ^ ProjectExecTransformer (19) - : +- ^ BroadcastHashJoinExecTransformer Inner (18) - : :- ^ ProjectExecTransformer (10) - : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : :- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ Scan parquet (2) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ BroadcastQueryStage (15) - : +- ColumnarBroadcastExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ Scan parquet (11) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ BroadcastQueryStage (23) - +- ColumnarBroadcastExchange (22) - +- ^ Scan parquet (20) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- BroadcastHashJoin Inner BuildRight (57) - :- Project (53) - : +- BroadcastHashJoin Inner BuildRight (52) - : :- Project (47) - : : +- BroadcastHashJoin Inner BuildRight (46) - : : :- Filter (41) - : : : +- Scan parquet (40) - : : +- BroadcastExchange (45) - : : +- Project (44) - : : +- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- BroadcastExchange (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -68,280 +72,296 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(3) ProjectExecTransformer +(4) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(5) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(4) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) Scan parquet +(23) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(21) WholeStageCodegenTransformer (X) +(24) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(25) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(22) ColumnarBroadcastExchange +(26) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(27) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(24) InputAdapter +(28) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(25) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(26) BroadcastHashJoinExecTransformer +(30) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(27) ProjectExecTransformer +(31) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(28) FlushableHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(29) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(30) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(31) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(40) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(42) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(43) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(44) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(46) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(48) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(49) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(50) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(51) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(52) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(53) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(54) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(55) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(56) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(59) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(60) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index 32cb5ccf87b3..219d4b7c14de 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ FilterExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ BroadcastHashJoinExecTransformer Inner (17) - :- ^ ProjectExecTransformer (9) - : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ BroadcastQueryStage (14) - +- ColumnarBroadcastExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ^ InputAdapter (34) + +- ^ ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- ^ InputAdapter (8) + : +- ^ BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildRight (41) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- BroadcastHashJoin Inner BuildRight (50) + :- Project (45) + : +- BroadcastHashJoin Inner BuildRight (44) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (49) + +- Project (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -60,240 +63,252 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(5) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(14) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [n_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [n_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(19) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(20) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(21) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(22) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(29) ColumnarExchange +(32) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [2]: [ps_partkey#X, value#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(36) Scan parquet +(39) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(40) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(41) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(42) Project +(45) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(45) Project +(48) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(46) BroadcastExchange +(49) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) Project +(51) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(49) HashAggregate +(52) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(50) Exchange +(53) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(52) Filter +(55) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(53) Exchange +(56) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(57) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index a9cea181da0f..985cbf4c3c59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin Inner BuildLeft (33) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -46,174 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(12) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(13) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(14) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(16) InputAdapter +(18) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(17) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(32) Project +(34) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(33) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(34) Project +(36) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(36) Exchange +(38) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 1963624ed312..4a3a239381ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ ProjectExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- BroadcastHashJoin LeftOuter BuildRight (41) - :- Scan parquet (36) - +- BroadcastExchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- BroadcastHashJoin LeftOuter BuildRight (42) + :- Scan parquet (37) + +- BroadcastExchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -62,220 +63,224 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) ProjectExecTransformer +(3) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(4) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(11) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(14) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [2]: [c_custkey#X, count#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(39) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(40) BroadcastExchange +(41) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(43) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(44) Exchange +(45) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index 1d64bb80886d..ee1fe0a3316c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ ProjectExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (22) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (31) - +- Exchange (30) - +- HashAggregate (29) - +- Project (28) - +- BroadcastHashJoin Inner BuildRight (27) - :- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- BroadcastExchange (26) - +- Filter (25) - +- Scan parquet (24) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) Scan parquet +(4) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(6) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(21) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(22) Filter +(24) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(23) Project +(25) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(26) BroadcastExchange +(28) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(28) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(29) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(30) Exchange +(32) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(31) HashAggregate +(33) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index d8563db5b941..076e82bd53e3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (17) - +- ^ RegularHashAggregateExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FlushableHashAggregateExecTransformer (9) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- BroadcastHashJoin Inner BuildLeft (38) - :- BroadcastExchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- BroadcastHashJoin Inner BuildLeft (40) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,182 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(18) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) BroadcastExchange +(32) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) BroadcastHashJoin +(40) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index d8664006f507..603ea771b167 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,62 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (41) - +- ^ SortExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ ProjectExecTransformer (27) - +- ^ FlushableHashAggregateExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ BroadcastHashJoinExecTransformer Inner (16) - :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ BroadcastQueryStage (6) - : +- ColumnarBroadcastExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ BroadcastQueryStage (13) - +- ColumnarBroadcastExchange (12) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- BroadcastHashJoin LeftAnti BuildRight (48) - : :- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- BroadcastExchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- BroadcastHashJoin Inner BuildRight (46) + :- BroadcastHashJoin LeftAnti BuildRight (42) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -66,278 +61,252 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(3) ProjectExecTransformer -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [s_suppkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [s_suppkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [s_suppkey#X] - -(8) InputIteratorTransformer -Input [1]: [s_suppkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] -(10) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(5) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(12) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(7) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(14) InputAdapter +(8) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(17) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) ProjectExecTransformer +(13) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(21) ColumnarExchange +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(23) InputAdapter +(17) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(29) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(31) InputAdapter +(25) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(34) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) ColumnarExchange +(29) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(37) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) SortExecTransformer +(33) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(40) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(41) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) Scan parquet +(36) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(43) Filter +(37) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(44) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(45) Filter +(39) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(46) Project +(40) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(47) BroadcastExchange +(41) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(48) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(49) Scan parquet +(43) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(44) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) BroadcastExchange +(45) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(52) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(47) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(48) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(49) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(50) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(54) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(55) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index 468f95bff9fd..c294e8e6dac1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,83 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- TakeOrderedAndProjectExecTransformer (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (26) - : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : :- ^ InputIteratorTransformer (6) - : : +- ^ InputAdapter (5) - : : +- ^ BroadcastQueryStage (4) - : : +- ColumnarBroadcastExchange (3) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) - : :- ^ Scan parquet (7) - : +- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ BroadcastQueryStage (21) - : +- ColumnarBroadcastExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ FilterExecTransformer (17) - : +- ^ RegularHashAggregateExecTransformer (16) - : +- ^ InputIteratorTransformer (15) - : +- ^ InputAdapter (14) - : +- ^ ShuffleQueryStage (13) - : +- ColumnarExchange (12) - : +- ^ ProjectExecTransformer (10) - : +- ^ FlushableHashAggregateExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35) - +- ColumnarBroadcastExchange (34) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) - :- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ BroadcastQueryStage (29) - +- ReusedExchange (28) + VeloxColumnarToRowExec (53) + +- TakeOrderedAndProjectExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- BroadcastHashJoin Inner BuildRight (77) - :- Project (65) - : +- BroadcastHashJoin Inner BuildLeft (64) - : :- BroadcastExchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- BroadcastHashJoin LeftSemi BuildRight (63) - : :- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastExchange (62) - : +- Project (61) - : +- Filter (60) - : +- HashAggregate (59) - : +- Exchange (58) - : +- HashAggregate (57) - : +- Scan parquet (56) - +- BroadcastExchange (76) - +- BroadcastHashJoin LeftSemi BuildRight (75) - :- Filter (67) - : +- Scan parquet (66) - +- BroadcastExchange (74) - +- Project (73) - +- Filter (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Scan parquet (68) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -87,375 +90,387 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) Scan parquet +(9) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(10) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [1]: [l_orderkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [l_orderkey#X] -(24) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(25) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(28) ReusedExchange [Reuses operator id: 20] +(30) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(31) ReusedExchange [Reuses operator id: 22] Output [1]: [l_orderkey#X] -(29) BroadcastQueryStage +(32) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [1]: [l_orderkey#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [1]: [l_orderkey#X] -(32) BroadcastHashJoinExecTransformer +(35) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(34) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(38) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) ProjectExecTransformer +(42) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(40) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(44) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(49) TakeOrderedAndProjectExecTransformer +(52) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(53) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(54) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(56) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(57) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(66) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(79) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(80) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(82) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(83) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 1be7fcaa15c3..267ae6add0ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -1,34 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (31) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (30) - +- Exchange (29) - +- HashAggregate (28) - +- Project (27) - +- BroadcastHashJoin Inner BuildRight (26) - :- Project (22) - : +- Filter (21) - : +- Scan parquet (20) - +- BroadcastExchange (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (32) + +- Exchange (31) + +- HashAggregate (30) + +- Project (29) + +- BroadcastHashJoin Inner BuildRight (28) + :- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- BroadcastExchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -38,140 +40,148 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) Scan parquet +(4) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(6) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [revenue#X] -(20) Scan parquet +(22) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(21) Filter +(23) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(22) Project +(24) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(23) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(24) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(25) BroadcastExchange +(27) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastHashJoin +(28) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(27) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(28) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(29) Exchange +(31) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(30) HashAggregate +(32) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(31) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index d19495f88198..1b4522028bb4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,103 +1,105 @@ == Physical Plan == -AdaptiveSparkPlan (105) +AdaptiveSparkPlan (107) +- == Final Plan == - VeloxColumnarToRowExec (68) - +- ^ SortExecTransformer (66) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ ShuffleQueryStage (63) - +- ColumnarExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ BroadcastHashJoinExecTransformer Inner (59) - :- ^ ProjectExecTransformer (51) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (50) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ AQEShuffleRead (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ BroadcastQueryStage (47) - : +- ColumnarBroadcastExchange (46) - : +- AQEShuffleRead (45) - : +- ShuffleQueryStage (44) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ BroadcastHashJoinExecTransformer Inner (40) - : :- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : :- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ BroadcastQueryStage (14) - : : +- ColumnarBroadcastExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ FilterExecTransformer (39) - : +- ^ ProjectExecTransformer (38) - : +- ^ RegularHashAggregateExecTransformer (37) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ ShuffleQueryStage (34) - : +- ColumnarExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FlushableHashAggregateExecTransformer (30) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (29) - : :- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ BroadcastQueryStage (26) - : +- ReusedExchange (25) - +- ^ InputIteratorTransformer (58) - +- ^ InputAdapter (57) - +- ^ BroadcastQueryStage (56) - +- ColumnarBroadcastExchange (55) - +- ^ ProjectExecTransformer (53) - +- ^ Scan parquet (52) + VeloxColumnarToRowExec (70) + +- ^ SortExecTransformer (68) + +- ^ InputIteratorTransformer (67) + +- ^ InputAdapter (66) + +- ^ ShuffleQueryStage (65) + +- ColumnarExchange (64) + +- ^ ProjectExecTransformer (62) + +- ^ BroadcastHashJoinExecTransformer Inner (61) + :- ^ ProjectExecTransformer (52) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : :- ^ InputIteratorTransformer (9) + : : +- ^ InputAdapter (8) + : : +- ^ AQEShuffleRead (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ BroadcastQueryStage (48) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : :- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : :- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- ^ InputAdapter (18) + : : +- ^ BroadcastQueryStage (17) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ NoopFilter (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ^ InputAdapter (39) + : +- ^ ShuffleQueryStage (38) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ NoopFilter (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30) + : +- ReusedExchange (29) + +- ^ InputIteratorTransformer (60) + +- ^ InputAdapter (59) + +- ^ BroadcastQueryStage (58) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == - Sort (104) - +- Exchange (103) - +- Project (102) - +- BroadcastHashJoin Inner BuildRight (101) - :- Project (96) - : +- ShuffledHashJoin LeftSemi BuildRight (95) - : :- Exchange (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- Exchange (94) - : +- Project (93) - : +- BroadcastHashJoin Inner BuildLeft (92) - : :- BroadcastExchange (79) - : : +- BroadcastHashJoin LeftSemi BuildRight (78) - : : :- Filter (73) - : : : +- Scan parquet (72) - : : +- BroadcastExchange (77) - : : +- Project (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Filter (91) - : +- HashAggregate (90) - : +- Exchange (89) - : +- HashAggregate (88) - : +- BroadcastHashJoin LeftSemi BuildRight (87) - : :- Project (82) - : : +- Filter (81) - : : +- Scan parquet (80) - : +- BroadcastExchange (86) - : +- Project (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (100) - +- Project (99) - +- Filter (98) - +- Scan parquet (97) + Sort (106) + +- Exchange (105) + +- Project (104) + +- BroadcastHashJoin Inner BuildRight (103) + :- Project (98) + : +- ShuffledHashJoin LeftSemi BuildRight (97) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (96) + : +- Project (95) + : +- BroadcastHashJoin Inner BuildLeft (94) + : :- BroadcastExchange (81) + : : +- BroadcastHashJoin LeftSemi BuildRight (80) + : : :- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (79) + : : +- Project (78) + : : +- Filter (77) + : : +- Scan parquet (76) + : +- Filter (93) + : +- HashAggregate (92) + : +- Exchange (91) + : +- HashAggregate (90) + : +- BroadcastHashJoin LeftSemi BuildRight (89) + : :- Project (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- BroadcastExchange (88) + : +- Project (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (102) + +- Project (101) + +- Filter (100) + +- Scan parquet (99) (1) Scan parquet @@ -107,452 +109,460 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) AQEShuffleRead +(7) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) Scan parquet +(11) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(12) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(14) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(23) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(24) ProjectExecTransformer +(27) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(28) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(25) ReusedExchange [Reuses operator id: 13] +(29) ReusedExchange [Reuses operator id: 16] Output [1]: [p_partkey#X] -(26) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(27) InputAdapter +(31) InputAdapter Input [1]: [p_partkey#X] -(28) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [p_partkey#X] -(29) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(31) ProjectExecTransformer +(35) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(33) ColumnarExchange +(37) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(38) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(35) InputAdapter +(39) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) InputIteratorTransformer +(40) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(38) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(39) FilterExecTransformer +(43) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(40) BroadcastHashJoinExecTransformer +(44) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(41) ProjectExecTransformer -Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(42) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: false - -(43) ColumnarExchange -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] - -(44) ShuffleQueryStage +(45) ProjectExecTransformer Output [1]: [ps_suppkey#X] -Arguments: X +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(45) AQEShuffleRead +(46) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] -Arguments: local +Arguments: false -(46) ColumnarBroadcastExchange +(47) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastQueryStage +(48) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(48) InputAdapter +(49) InputAdapter Input [1]: [ps_suppkey#X] -(49) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(50) BroadcastHashJoinExecTransformer +(51) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(51) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(52) Scan parquet +(53) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(53) ProjectExecTransformer +(54) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(55) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(54) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(55) ColumnarBroadcastExchange +(57) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastQueryStage +(58) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [1]: [n_nationkey#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [1]: [n_nationkey#X] -(59) BroadcastHashJoinExecTransformer +(61) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(60) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(62) ColumnarExchange +(64) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(63) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(64) InputAdapter +(66) InputAdapter Input [2]: [s_name#X, s_address#X] -(65) InputIteratorTransformer +(67) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(66) SortExecTransformer +(68) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(67) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(68) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(69) Scan parquet +(71) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(71) Exchange +(73) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(74) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(73) Filter +(75) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(74) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(80) Scan parquet +(82) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(81) Filter +(83) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(82) Project +(84) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(85) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(86) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(88) HashAggregate +(90) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(91) Filter +(93) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(92) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(93) Project +(95) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(94) Exchange +(96) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(96) Project +(98) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(97) Scan parquet +(99) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(98) Filter +(100) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(99) Project +(101) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(100) BroadcastExchange +(102) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(102) Project +(104) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(103) Exchange +(105) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Sort +(106) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(105) AdaptiveSparkPlan +(107) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index faf7f5799f34..9c09032689eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,86 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- TakeOrderedAndProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ InputIteratorTransformer (6) - : : : +- ^ InputAdapter (5) - : : : +- ^ BroadcastQueryStage (4) - : : : +- ColumnarBroadcastExchange (3) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) - : : : :- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ BroadcastQueryStage (12) - : : : +- ColumnarBroadcastExchange (11) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ColumnarBroadcastExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (59) + +- TakeOrderedAndProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (76) - : +- BroadcastHashJoin Inner BuildRight (75) - : :- Project (70) - : : +- BroadcastHashJoin Inner BuildLeft (69) - : : :- BroadcastExchange (57) - : : : +- Filter (56) - : : : +- Scan parquet (55) - : : +- BroadcastHashJoin LeftAnti BuildRight (68) - : : :- BroadcastHashJoin LeftSemi BuildRight (63) - : : : :- Project (60) - : : : : +- Filter (59) - : : : : +- Scan parquet (58) - : : : +- BroadcastExchange (62) - : : : +- Scan parquet (61) - : : +- BroadcastExchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- BroadcastExchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- BroadcastExchange (80) - +- Project (79) - +- Filter (78) - +- Scan parquet (77) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -90,386 +95,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(9) Scan parquet +(11) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(12) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(16) Scan parquet +(18) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(17) ProjectExecTransformer +(19) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(20) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(23) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(24) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(25) ProjectExecTransformer +(28) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(26) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(27) ProjectExecTransformer +(30) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(31) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(28) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastQueryStage +(34) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [1]: [o_orderkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [1]: [o_orderkey#X] -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(36) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(37) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(44) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [1]: [n_nationkey#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [1]: [n_nationkey#X] -(42) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(43) ProjectExecTransformer +(48) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(44) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(45) ProjectExecTransformer +(50) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(46) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(47) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(49) InputAdapter +(54) InputAdapter Input [2]: [s_name#X, count#X] -(50) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(51) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(53) TakeOrderedAndProjectExecTransformer +(58) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(54) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(55) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(56) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(57) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(58) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(59) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(60) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(61) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(62) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(64) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(65) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(66) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(67) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(70) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(71) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(73) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(74) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(76) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(77) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(78) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(79) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(80) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(83) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(84) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(86) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(87) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index e33dc6b7fb9b..3dc92be2d96e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,39 +1,40 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (38) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ SortExecTransformer (23) - +- ^ InputIteratorTransformer (22) - +- ^ InputAdapter (21) - +- ^ ShuffleQueryStage (20) - +- ColumnarExchange (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (7) - +- ^ InputAdapter (6) - +- ^ BroadcastQueryStage (5) - +- ColumnarBroadcastExchange (4) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (36) - +- Exchange (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin LeftAnti BuildRight (30) - :- Filter (27) - : +- Scan parquet (26) - +- BroadcastExchange (29) - +- Scan parquet (28) + Sort (37) + +- Exchange (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin LeftAnti BuildRight (31) + :- Filter (28) + : +- Scan parquet (27) + +- BroadcastExchange (30) + +- Scan parquet (29) (1) Scan parquet @@ -43,160 +44,164 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [o_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [o_custkey#X] -(8) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(10) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(14) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(19) ColumnarExchange +(20) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) InputAdapter +(22) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(22) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) SortExecTransformer +(24) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(25) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(26) Scan parquet +(27) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(27) Filter +(28) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(28) Scan parquet +(29) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(29) BroadcastExchange +(30) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(31) Project +(32) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(32) HashAggregate +(33) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(35) Exchange +(36) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(37) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(37) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index e6304542a9ce..f962dbdd765f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,52 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (31) - +- TakeOrderedAndProjectExecTransformer (30) - +- ^ ProjectExecTransformer (28) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ ProjectExecTransformer (10) - : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ BroadcastQueryStage (15) - +- ColumnarBroadcastExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ Scan parquet (11) + VeloxColumnarToRowExec (34) + +- TakeOrderedAndProjectExecTransformer (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Project (45) - +- BroadcastHashJoin Inner BuildRight (44) - :- Project (39) - : +- BroadcastHashJoin Inner BuildLeft (38) - : :- BroadcastExchange (35) - : : +- Project (34) - : : +- Filter (33) - : : +- Scan parquet (32) - : +- Filter (37) - : +- Scan parquet (36) - +- BroadcastExchange (43) - +- Project (42) - +- Filter (41) - +- Scan parquet (40) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -56,222 +59,234 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(24) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(26) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(28) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(30) TakeOrderedAndProjectExecTransformer +(33) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(31) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(32) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(33) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(34) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(35) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(36) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(38) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(41) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(42) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(45) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(46) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(47) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(49) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 5ea636f9ac11..e50973eb5abf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- ^ InputAdapter (10) + +- ^ BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin LeftSemi BuildRight (35) - :- Project (30) - : +- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin LeftSemi BuildRight (37) + :- Project (32) + : +- Filter (31) + : +- Scan parquet (30) + +- BroadcastExchange (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,182 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) Scan parquet +(4) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(4) ProjectExecTransformer +(5) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(6) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(5) WholeStageCodegenTransformer (X) +(7) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(8) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(9) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(8) InputAdapter +(10) InputAdapter Input [1]: [l_orderkey#X] -(9) InputIteratorTransformer +(11) InputIteratorTransformer Input [1]: [l_orderkey#X] -(10) BroadcastHashJoinExecTransformer +(12) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(12) FlushableHashAggregateExecTransformer +(14) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(13) ProjectExecTransformer +(15) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(14) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(15) ColumnarExchange +(17) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(28) Scan parquet +(30) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(29) Filter +(31) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(30) Project +(32) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(31) Scan parquet +(33) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(32) Filter +(34) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(33) Project +(35) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(34) BroadcastExchange +(36) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(37) HashAggregate +(39) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(38) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index 377110d0e7a1..274c176ff8c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,92 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ SortExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (62) - : : : : : +- Filter (61) - : : : : : +- Scan parquet (60) - : : : : +- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Project (71) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -96,414 +102,438 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(10) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(15) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(24) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(27) Scan parquet +(31) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(33) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(36) ProjectExecTransformer +(41) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(42) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(37) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(44) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(45) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(40) InputAdapter +(46) InputAdapter Input [1]: [r_regionkey#X] -(41) InputIteratorTransformer +(47) InputIteratorTransformer Input [1]: [r_regionkey#X] -(42) BroadcastHashJoinExecTransformer +(48) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(43) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(44) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(45) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(46) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(47) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(55) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(52) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(53) ColumnarExchange +(59) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(55) InputAdapter +(61) InputAdapter Input [2]: [n_name#X, revenue#X] -(56) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(57) SortExecTransformer +(63) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(60) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(62) BroadcastExchange +(68) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(65) Project +(71) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(66) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(67) Project +(73) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(70) BroadcastExchange +(76) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(78) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(79) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(80) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(81) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(77) Project +(83) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(84) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(80) BroadcastExchange +(86) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(83) Scan parquet +(89) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(84) Filter +(90) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(85) Project +(91) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(86) BroadcastExchange +(92) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(88) Project +(94) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) HashAggregate +(95) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) Exchange +(96) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(97) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(92) Exchange +(98) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(99) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index cc87fe6d2bbf..8d1a71e9751e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index 8716498ef067..3d5b6496fe27 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,87 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50) - +- ColumnarExchange (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (33) - : +- ^ BroadcastHashJoinExecTransformer Inner (32) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ ProjectExecTransformer (17) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) - : : : :- ^ ProjectExecTransformer (9) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (15) - : : : +- ^ InputAdapter (14) - : : : +- ^ BroadcastQueryStage (13) - : : : +- ColumnarBroadcastExchange (12) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ BroadcastQueryStage (21) - : : +- ColumnarBroadcastExchange (20) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ BroadcastQueryStage (29) - : +- ColumnarBroadcastExchange (28) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35) - +- ReusedExchange (34) + VeloxColumnarToRowExec (60) + +- ^ SortExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- ^ InputAdapter (17) + : : : +- ^ BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- ^ InputAdapter (26) + : : +- ^ BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (87) - +- Exchange (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (77) - : +- BroadcastHashJoin Inner BuildRight (76) - : :- Project (72) - : : +- BroadcastHashJoin Inner BuildRight (71) - : : :- Project (67) - : : : +- BroadcastHashJoin Inner BuildRight (66) - : : : :- Project (62) - : : : : +- BroadcastHashJoin Inner BuildLeft (61) - : : : : :- BroadcastExchange (58) - : : : : : +- Filter (57) - : : : : : +- Scan parquet (56) - : : : : +- Filter (60) - : : : : +- Scan parquet (59) - : : : +- BroadcastExchange (65) - : : : +- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (75) - : +- Filter (74) - : +- Scan parquet (73) - +- BroadcastExchange (80) - +- Filter (79) - +- Scan parquet (78) + Sort (92) + +- Exchange (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (63) + : : : : : +- Filter (62) + : : : : : +- Scan parquet (61) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -91,386 +96,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) BroadcastHashJoinExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(13) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(14) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) BroadcastHashJoinExecTransformer +(19) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(17) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(18) Scan parquet +(21) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(19) WholeStageCodegenTransformer (X) +(22) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(23) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(24) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(21) BroadcastQueryStage +(25) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(22) InputAdapter +(26) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(23) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(24) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(25) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(26) Scan parquet +(30) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(27) WholeStageCodegenTransformer (X) +(31) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(32) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(28) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastQueryStage +(34) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(30) InputAdapter +(35) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(31) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(32) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(38) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(34) ReusedExchange [Reuses operator id: 28] +(39) ReusedExchange [Reuses operator id: 33] Output [2]: [n_nationkey#X, n_name#X] -(35) BroadcastQueryStage +(40) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(36) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(37) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(38) BroadcastHashJoinExecTransformer +(43) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(39) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(40) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(48) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(49) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(50) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(51) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(48) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(49) ColumnarExchange +(54) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(55) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(51) InputAdapter +(56) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(52) InputIteratorTransformer +(57) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(53) SortExecTransformer +(58) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(54) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(56) Scan parquet +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(58) BroadcastExchange +(63) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(59) Scan parquet +(64) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(61) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(62) Project +(67) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(63) Scan parquet +(68) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(64) Filter +(69) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(65) BroadcastExchange +(70) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(66) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(67) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(73) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(70) BroadcastExchange +(75) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(72) Project +(77) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(73) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(74) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(75) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(77) Project +(82) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(78) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(79) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(82) Project +(87) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(88) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(86) Exchange +(91) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Sort +(92) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 94e003e55d93..9489452b5272 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,117 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ SortExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ShuffleQueryStage (64) - +- ColumnarExchange (63) - +- ^ ProjectExecTransformer (61) - +- ^ FlushableHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ BroadcastHashJoinExecTransformer Inner (58) - :- ^ ProjectExecTransformer (50) - : +- ^ BroadcastHashJoinExecTransformer Inner (49) - : :- ^ ProjectExecTransformer (42) - : : +- ^ BroadcastHashJoinExecTransformer Inner (41) - : : :- ^ ProjectExecTransformer (34) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : : : :- ^ ProjectExecTransformer (26) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : : : :- ^ ProjectExecTransformer (18) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : : : :- ^ ProjectExecTransformer (10) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ BroadcastQueryStage (5) - : : : : : : : +- ColumnarBroadcastExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (16) - : : : : : +- ^ InputAdapter (15) - : : : : : +- ^ BroadcastQueryStage (14) - : : : : : +- ColumnarBroadcastExchange (13) - : : : : : +- ^ Scan parquet (11) - : : : : +- ^ InputIteratorTransformer (24) - : : : : +- ^ InputAdapter (23) - : : : : +- ^ BroadcastQueryStage (22) - : : : : +- ColumnarBroadcastExchange (21) - : : : : +- ^ Scan parquet (19) - : : : +- ^ InputIteratorTransformer (32) - : : : +- ^ InputAdapter (31) - : : : +- ^ BroadcastQueryStage (30) - : : : +- ColumnarBroadcastExchange (29) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (40) - : : +- ^ InputAdapter (39) - : : +- ^ BroadcastQueryStage (38) - : : +- ColumnarBroadcastExchange (37) - : : +- ^ Scan parquet (35) - : +- ^ InputIteratorTransformer (48) - : +- ^ InputAdapter (47) - : +- ^ BroadcastQueryStage (46) - : +- ColumnarBroadcastExchange (45) - : +- ^ Scan parquet (43) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ BroadcastQueryStage (55) - +- ColumnarBroadcastExchange (54) - +- ^ ProjectExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (84) + +- ^ SortExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- ^ InputAdapter (18) + : : : : : +- ^ BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- ^ InputAdapter (27) + : : : : +- ^ BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- ^ InputAdapter (36) + : : : +- ^ BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- ^ InputAdapter (54) + : +- ^ BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- BroadcastHashJoin Inner BuildRight (114) - :- Project (109) - : +- BroadcastHashJoin Inner BuildRight (108) - : :- Project (104) - : : +- BroadcastHashJoin Inner BuildRight (103) - : : :- Project (99) - : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : :- Project (94) - : : : : +- BroadcastHashJoin Inner BuildRight (93) - : : : : :- Project (89) - : : : : : +- BroadcastHashJoin Inner BuildRight (88) - : : : : : :- Project (84) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) - : : : : : : :- BroadcastExchange (80) - : : : : : : : +- Project (79) - : : : : : : : +- Filter (78) - : : : : : : : +- Scan parquet (77) - : : : : : : +- Filter (82) - : : : : : : +- Scan parquet (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- Filter (86) - : : : : : +- Scan parquet (85) - : : : : +- BroadcastExchange (92) - : : : : +- Filter (91) - : : : : +- Scan parquet (90) - : : : +- BroadcastExchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- BroadcastExchange (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- BroadcastExchange (107) - : +- Filter (106) - : +- Scan parquet (105) - +- BroadcastExchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (128) + +- Exchange (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- BroadcastHashJoin Inner BuildRight (122) + :- Project (117) + : +- BroadcastHashJoin Inner BuildRight (116) + : :- Project (112) + : : +- BroadcastHashJoin Inner BuildRight (111) + : : :- Project (107) + : : : +- BroadcastHashJoin Inner BuildRight (106) + : : : :- Project (102) + : : : : +- BroadcastHashJoin Inner BuildRight (101) + : : : : :- Project (97) + : : : : : +- BroadcastHashJoin Inner BuildRight (96) + : : : : : :- Project (92) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) + : : : : : : :- BroadcastExchange (88) + : : : : : : : +- Project (87) + : : : : : : : +- Filter (86) + : : : : : : : +- Scan parquet (85) + : : : : : : +- Filter (90) + : : : : : : +- Scan parquet (89) + : : : : : +- BroadcastExchange (95) + : : : : : +- Filter (94) + : : : : : +- Scan parquet (93) + : : : : +- BroadcastExchange (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- BroadcastExchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- BroadcastExchange (110) + : : +- Filter (109) + : : +- Scan parquet (108) + : +- BroadcastExchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- BroadcastExchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,534 +129,566 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(43) Scan parquet +(49) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(44) WholeStageCodegenTransformer (X) +(50) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(51) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(45) ColumnarBroadcastExchange +(52) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastQueryStage +(53) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(47) InputAdapter +(54) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(48) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastHashJoinExecTransformer +(56) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(50) ProjectExecTransformer +(57) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(51) Scan parquet +(58) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(52) ProjectExecTransformer +(59) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(60) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(53) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(54) ColumnarBroadcastExchange +(62) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(55) BroadcastQueryStage +(63) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(56) InputAdapter +(64) InputAdapter Input [1]: [r_regionkey#X] -(57) InputIteratorTransformer +(65) InputIteratorTransformer Input [1]: [r_regionkey#X] -(58) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(59) ProjectExecTransformer +(67) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(60) FlushableHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(61) ProjectExecTransformer +(69) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(62) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(63) ColumnarExchange +(71) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(64) ShuffleQueryStage +(72) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(65) InputAdapter +(73) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(66) InputIteratorTransformer +(74) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(67) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(68) ProjectExecTransformer +(76) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(69) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(70) ColumnarExchange +(78) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(72) InputAdapter +(80) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(73) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(74) SortExecTransformer +(82) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(75) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(76) VeloxColumnarToRowExec +(84) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(77) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(78) Filter +(86) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(79) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(80) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(90) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(84) Project +(92) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(85) Scan parquet +(93) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(94) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(95) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(97) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(91) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(92) BroadcastExchange +(100) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(93) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(94) Project +(102) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Scan parquet +(103) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(104) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) BroadcastExchange +(105) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(98) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(99) Project +(107) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(100) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(101) Filter +(109) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(102) BroadcastExchange +(110) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(103) BroadcastHashJoin +(111) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(112) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(105) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(106) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(107) BroadcastExchange +(115) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(108) BroadcastHashJoin +(116) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) Project +(117) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(110) Scan parquet +(118) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(111) Filter +(119) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(112) Project +(120) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(113) BroadcastExchange +(121) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(114) BroadcastHashJoin +(122) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(115) Project +(123) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(116) HashAggregate +(124) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(117) Exchange +(125) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(126) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(119) Exchange +(127) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(128) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index d337c7f9e468..29e6d2f72f57 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,90 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ SortExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (64) + +- ^ SortExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ^ InputAdapter (60) + +- ^ ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (91) - +- Exchange (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildRight (75) - : : :- Project (71) - : : : +- BroadcastHashJoin Inner BuildRight (70) - : : : :- Project (66) - : : : : +- BroadcastHashJoin Inner BuildLeft (65) - : : : : :- BroadcastExchange (62) - : : : : : +- Project (61) - : : : : : +- Filter (60) - : : : : : +- Scan parquet (59) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (84) - +- Filter (83) - +- Scan parquet (82) + Sort (97) + +- Exchange (96) + +- HashAggregate (95) + +- Exchange (94) + +- HashAggregate (93) + +- Project (92) + +- BroadcastHashJoin Inner BuildRight (91) + :- Project (87) + : +- BroadcastHashJoin Inner BuildRight (86) + : :- Project (82) + : : +- BroadcastHashJoin Inner BuildRight (81) + : : :- Project (77) + : : : +- BroadcastHashJoin Inner BuildRight (76) + : : : :- Project (72) + : : : : +- BroadcastHashJoin Inner BuildLeft (71) + : : : : :- BroadcastExchange (68) + : : : : : +- Project (67) + : : : : : +- Filter (66) + : : : : : +- Scan parquet (65) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (75) + : : : +- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- BroadcastExchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -94,406 +100,430 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(43) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(52) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(56) SortExecTransformer +(62) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(59) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(60) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(61) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(62) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(66) Project +(72) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(67) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(68) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(69) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(71) Project +(77) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(72) Scan parquet +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(73) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(74) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(75) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(76) Project +(82) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(77) Scan parquet +(83) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(84) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(79) BroadcastExchange +(85) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(87) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(82) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(84) BroadcastExchange +(90) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(85) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(92) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(87) HashAggregate +(93) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(88) Exchange +(94) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(95) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(90) Exchange +(96) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Sort +(97) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(92) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index c5d58658acd5..74f79bd3ee64 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index c599bb00a7bb..276588749734 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,64 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ FlushableHashAggregateExecTransformer (28) - +- ^ ProjectExecTransformer (27) - +- ^ BroadcastHashJoinExecTransformer Inner (26) - :- ^ ProjectExecTransformer (19) - : +- ^ BroadcastHashJoinExecTransformer Inner (18) - : :- ^ ProjectExecTransformer (10) - : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : :- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ Scan parquet (2) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ BroadcastQueryStage (15), Statistics(X) - : +- ColumnarBroadcastExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ Scan parquet (11) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ BroadcastQueryStage (23), Statistics(X) - +- ColumnarBroadcastExchange (22) - +- ^ Scan parquet (20) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- BroadcastHashJoin Inner BuildRight (57) - :- Project (53) - : +- BroadcastHashJoin Inner BuildRight (52) - : :- Project (47) - : : +- BroadcastHashJoin Inner BuildRight (46) - : : :- Filter (41) - : : : +- Scan parquet (40) - : : +- BroadcastExchange (45) - : : +- Project (44) - : : +- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- BroadcastExchange (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -68,280 +72,296 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(3) ProjectExecTransformer +(4) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(5) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(4) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) Scan parquet +(23) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(21) WholeStageCodegenTransformer (X) +(24) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(25) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(22) ColumnarBroadcastExchange +(26) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(27) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(24) InputAdapter +(28) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(25) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(26) BroadcastHashJoinExecTransformer +(30) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(27) ProjectExecTransformer +(31) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(28) FlushableHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(29) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(30) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(31) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(40) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(42) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(43) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(44) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(46) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(47) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(48) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(49) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(50) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(51) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(52) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(53) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(54) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(55) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(56) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(59) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(60) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 459c52738acb..54f535dfbfb6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ FilterExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ BroadcastHashJoinExecTransformer Inner (17) - :- ^ ProjectExecTransformer (9) - : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ BroadcastQueryStage (14), Statistics(X) - +- ColumnarBroadcastExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ^ InputAdapter (34) + +- ^ ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- ^ InputAdapter (8) + : +- ^ BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildRight (41) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- BroadcastHashJoin Inner BuildRight (50) + :- Project (45) + : +- BroadcastHashJoin Inner BuildRight (44) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (49) + +- Project (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -60,464 +63,481 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(5) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(14) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [n_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [n_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(19) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(20) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(21) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(22) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(29) ColumnarExchange +(32) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [2]: [ps_partkey#X, value#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(36) Scan parquet +(39) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(40) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(41) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(42) Project +(45) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(45) Project +(48) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(46) BroadcastExchange +(49) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) Project +(51) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(49) HashAggregate +(52) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(50) Exchange +(53) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(52) Filter +(55) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(53) Exchange +(56) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(57) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 27 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (95) +Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ FlushableHashAggregateExecTransformer (69) - +- ^ ProjectExecTransformer (68) - +- ^ BroadcastHashJoinExecTransformer Inner (67) - :- ^ ProjectExecTransformer (62) - : +- ^ BroadcastHashJoinExecTransformer Inner (61) - : :- ^ Scan parquet (56) - : +- ^ InputIteratorTransformer (60) - : +- ^ InputAdapter (59) - : +- ^ BroadcastQueryStage (58), Statistics(X) - : +- ReusedExchange (57) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ BroadcastQueryStage (64), Statistics(X) - +- ReusedExchange (63) + VeloxColumnarToRowExec (82) + +- ^ ProjectExecTransformer (80) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : :- ^ NoopFilter (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == - HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Project (91) - +- BroadcastHashJoin Inner BuildRight (90) - :- Project (85) - : +- BroadcastHashJoin Inner BuildRight (84) - : :- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (83) - : +- Filter (82) - : +- Scan parquet (81) - +- BroadcastExchange (89) - +- Project (88) - +- Filter (87) - +- Scan parquet (86) - - -(56) Scan parquet + HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Filter (84) + : : +- Scan parquet (83) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) + + +(59) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) ReusedExchange [Reuses operator id: 4] +(60) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(61) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(58) BroadcastQueryStage +(62) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(59) InputAdapter +(63) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(60) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(65) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(62) ProjectExecTransformer +(66) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(63) ReusedExchange [Reuses operator id: 13] +(67) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(64) BroadcastQueryStage +(68) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(65) InputAdapter +(69) InputAdapter Input [1]: [n_nationkey#X] -(66) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [n_nationkey#X] -(67) BroadcastHashJoinExecTransformer +(71) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(68) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(69) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(75) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(77) InputAdapter Input [2]: [sum#X, isEmpty#X] -(74) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(76) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(77) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(78) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(79) Scan parquet +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) Filter +(84) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(81) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(82) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(83) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(84) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(85) Project +(89) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(86) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(87) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(88) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(89) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(91) Project +(95) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(92) HashAggregate +(96) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(93) Exchange +(97) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(98) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(95) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index d0dca8b40c03..1f1fcd3fb577 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin Inner BuildLeft (33) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -46,174 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(12) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(13) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(14) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(16) InputAdapter +(18) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(17) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(32) Project +(34) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(33) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(34) Project +(36) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(36) Exchange +(38) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index a9269637304d..e67819b82860 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ ProjectExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- BroadcastHashJoin LeftOuter BuildRight (41) - :- Scan parquet (36) - +- BroadcastExchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- BroadcastHashJoin LeftOuter BuildRight (42) + :- Scan parquet (37) + +- BroadcastExchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -62,220 +63,224 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) ProjectExecTransformer +(3) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(4) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(11) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(14) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [2]: [c_custkey#X, count#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(39) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(40) BroadcastExchange +(41) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(43) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(44) Exchange +(45) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index 2b4a4b697d1d..eb04b6c18271 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ ProjectExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (22) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (31) - +- Exchange (30) - +- HashAggregate (29) - +- Project (28) - +- BroadcastHashJoin Inner BuildRight (27) - :- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- BroadcastExchange (26) - +- Filter (25) - +- Scan parquet (24) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) Scan parquet +(4) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(6) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(21) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(22) Filter +(24) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(23) Project +(25) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(26) BroadcastExchange +(28) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(28) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(29) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(30) Exchange +(32) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(31) HashAggregate +(33) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 3bb88d987670..5c6304211102 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (41) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- AQEShuffleRead (23) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (17) - +- ^ RegularHashAggregateExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FlushableHashAggregateExecTransformer (9) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (26) + +- AQEShuffleRead (25) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Filter (34) - +- HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- Filter (29) - +- Scan parquet (28) + Sort (40) + +- Exchange (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Filter (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -46,324 +48,337 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(18) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(23) AQEShuffleRead +(25) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(24) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) Scan parquet +(27) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(27) BroadcastExchange +(29) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(30) Project +(32) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(32) Exchange +(34) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(34) Filter +(36) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(36) Project +(38) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(37) Exchange +(39) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(40) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(41) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 17 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ^ InputAdapter (47) - +- ^ ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ ProjectExecTransformer (43) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (57) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ NoopFilter (43) + +- ^ Scan parquet (42) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Exchange (59) - +- HashAggregate (58) - +- Project (57) - +- Filter (56) - +- Scan parquet (55) + HashAggregate (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- Filter (59) + +- Scan parquet (58) -(40) Scan parquet +(42) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(44) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(42) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(43) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(45) ColumnarExchange +(48) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(49) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(47) InputAdapter +(50) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) InputIteratorTransformer +(51) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(50) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(53) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(54) VeloxColumnarToRowExec +(57) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(55) Scan parquet +(58) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(56) Filter +(59) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(57) Project +(60) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(58) HashAggregate +(61) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(59) Exchange +(62) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) HashAggregate +(63) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(61) HashAggregate +(64) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(63) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index 2d1099920a41..9a4005e3da85 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,62 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (41) - +- ^ SortExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ ProjectExecTransformer (27) - +- ^ FlushableHashAggregateExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ BroadcastHashJoinExecTransformer Inner (16) - :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ BroadcastQueryStage (6), Statistics(X) - : +- ColumnarBroadcastExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ BroadcastQueryStage (13), Statistics(X) - +- ColumnarBroadcastExchange (12) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- BroadcastHashJoin LeftAnti BuildRight (48) - : :- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- BroadcastExchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- BroadcastHashJoin Inner BuildRight (46) + :- BroadcastHashJoin LeftAnti BuildRight (42) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -66,278 +61,252 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(3) ProjectExecTransformer -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [s_suppkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [s_suppkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [s_suppkey#X] - -(8) InputIteratorTransformer -Input [1]: [s_suppkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join condition: None +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] -(10) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(5) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(12) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(7) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(14) InputAdapter +(8) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(17) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) ProjectExecTransformer +(13) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(21) ColumnarExchange +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(23) InputAdapter +(17) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(29) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(31) InputAdapter +(25) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(34) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) ColumnarExchange +(29) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(37) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) SortExecTransformer +(33) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(40) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(41) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) Scan parquet +(36) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(43) Filter +(37) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(44) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(45) Filter +(39) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(46) Project +(40) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(47) BroadcastExchange +(41) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(48) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(49) Scan parquet +(43) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(44) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) BroadcastExchange +(45) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(52) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(47) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(48) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(49) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(50) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(54) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(55) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 48f5a91500b2..5a941687c525 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,83 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- TakeOrderedAndProjectExecTransformer (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (26) - : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : :- ^ InputIteratorTransformer (6) - : : +- ^ InputAdapter (5) - : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : +- ColumnarBroadcastExchange (3) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) - : :- ^ Scan parquet (7) - : +- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ BroadcastQueryStage (21), Statistics(X) - : +- ColumnarBroadcastExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ FilterExecTransformer (17) - : +- ^ RegularHashAggregateExecTransformer (16) - : +- ^ InputIteratorTransformer (15) - : +- ^ InputAdapter (14) - : +- ^ ShuffleQueryStage (13), Statistics(X) - : +- ColumnarExchange (12) - : +- ^ ProjectExecTransformer (10) - : +- ^ FlushableHashAggregateExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ColumnarBroadcastExchange (34) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) - :- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ BroadcastQueryStage (29), Statistics(X) - +- ReusedExchange (28) + VeloxColumnarToRowExec (53) + +- TakeOrderedAndProjectExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- BroadcastHashJoin Inner BuildRight (77) - :- Project (65) - : +- BroadcastHashJoin Inner BuildLeft (64) - : :- BroadcastExchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- BroadcastHashJoin LeftSemi BuildRight (63) - : :- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastExchange (62) - : +- Project (61) - : +- Filter (60) - : +- HashAggregate (59) - : +- Exchange (58) - : +- HashAggregate (57) - : +- Scan parquet (56) - +- BroadcastExchange (76) - +- BroadcastHashJoin LeftSemi BuildRight (75) - :- Filter (67) - : +- Scan parquet (66) - +- BroadcastExchange (74) - +- Project (73) - +- Filter (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Scan parquet (68) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -87,375 +90,387 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) Scan parquet +(9) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(10) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [1]: [l_orderkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [l_orderkey#X] -(24) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(25) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(28) ReusedExchange [Reuses operator id: 20] +(30) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(31) ReusedExchange [Reuses operator id: 22] Output [1]: [l_orderkey#X] -(29) BroadcastQueryStage +(32) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [1]: [l_orderkey#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [1]: [l_orderkey#X] -(32) BroadcastHashJoinExecTransformer +(35) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(34) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(38) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(39) ProjectExecTransformer +(42) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(40) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(44) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(49) TakeOrderedAndProjectExecTransformer +(52) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(53) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(54) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(56) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(57) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(64) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(66) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(79) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(80) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(82) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(83) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 603998ea792a..244f1c6ffd89 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -1,34 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (31) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (30) - +- Exchange (29) - +- HashAggregate (28) - +- Project (27) - +- BroadcastHashJoin Inner BuildRight (26) - :- Project (22) - : +- Filter (21) - : +- Scan parquet (20) - +- BroadcastExchange (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (32) + +- Exchange (31) + +- HashAggregate (30) + +- Project (29) + +- BroadcastHashJoin Inner BuildRight (28) + :- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- BroadcastExchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -38,140 +40,148 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) Scan parquet +(4) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(6) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [revenue#X] -(20) Scan parquet +(22) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(21) Filter +(23) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(22) Project +(24) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(23) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(24) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(25) BroadcastExchange +(27) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastHashJoin +(28) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(27) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(28) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(29) Exchange +(31) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(30) HashAggregate +(32) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(31) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index c0c669bbce45..fd6deaabc6f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,101 +1,103 @@ == Physical Plan == -AdaptiveSparkPlan (102) +AdaptiveSparkPlan (104) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- AQEShuffleRead (64) - +- ShuffleQueryStage (63), Statistics(X) - +- ColumnarExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ BroadcastHashJoinExecTransformer Inner (59) - :- ^ ProjectExecTransformer (51) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (50) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ AQEShuffleRead (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ BroadcastQueryStage (47), Statistics(X) - : +- ColumnarBroadcastExchange (46) - : +- AQEShuffleRead (45) - : +- ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ BroadcastHashJoinExecTransformer Inner (40) - : :- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20), Statistics(X) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : :- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : +- ColumnarBroadcastExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ Scan parquet (10) - : +- ^ FilterExecTransformer (39) - : +- ^ ProjectExecTransformer (38) - : +- ^ RegularHashAggregateExecTransformer (37) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ ShuffleQueryStage (34), Statistics(X) - : +- ColumnarExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FlushableHashAggregateExecTransformer (30) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (29) - : :- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ BroadcastQueryStage (26), Statistics(X) - : +- ReusedExchange (25) - +- ^ InputIteratorTransformer (58) - +- ^ InputAdapter (57) - +- ^ BroadcastQueryStage (56), Statistics(X) - +- ColumnarBroadcastExchange (55) - +- ^ ProjectExecTransformer (53) - +- ^ Scan parquet (52) + VeloxColumnarToRowExec (67) + +- AQEShuffleRead (66) + +- ShuffleQueryStage (65), Statistics(X) + +- ColumnarExchange (64) + +- ^ ProjectExecTransformer (62) + +- ^ BroadcastHashJoinExecTransformer Inner (61) + :- ^ ProjectExecTransformer (52) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : :- ^ InputIteratorTransformer (9) + : : +- ^ InputAdapter (8) + : : +- ^ AQEShuffleRead (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ BroadcastQueryStage (48), Statistics(X) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : :- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : :- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- ^ InputAdapter (18) + : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ NoopFilter (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ^ InputAdapter (39) + : +- ^ ShuffleQueryStage (38), Statistics(X) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ NoopFilter (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ReusedExchange (29) + +- ^ InputIteratorTransformer (60) + +- ^ InputAdapter (59) + +- ^ BroadcastQueryStage (58), Statistics(X) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == - Sort (101) - +- Exchange (100) - +- Project (99) - +- BroadcastHashJoin Inner BuildRight (98) - :- Project (93) - : +- ShuffledHashJoin LeftSemi BuildRight (92) - : :- Exchange (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Exchange (91) - : +- Project (90) - : +- BroadcastHashJoin Inner BuildLeft (89) - : :- BroadcastExchange (76) - : : +- BroadcastHashJoin LeftSemi BuildRight (75) - : : :- Filter (70) - : : : +- Scan parquet (69) - : : +- BroadcastExchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Filter (88) - : +- HashAggregate (87) - : +- Exchange (86) - : +- HashAggregate (85) - : +- BroadcastHashJoin LeftSemi BuildRight (84) - : :- Project (79) - : : +- Filter (78) - : : +- Scan parquet (77) - : +- BroadcastExchange (83) - : +- Project (82) - : +- Filter (81) - : +- Scan parquet (80) - +- BroadcastExchange (97) - +- Project (96) - +- Filter (95) - +- Scan parquet (94) + Sort (103) + +- Exchange (102) + +- Project (101) + +- BroadcastHashJoin Inner BuildRight (100) + :- Project (95) + : +- ShuffledHashJoin LeftSemi BuildRight (94) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Exchange (93) + : +- Project (92) + : +- BroadcastHashJoin Inner BuildLeft (91) + : :- BroadcastExchange (78) + : : +- BroadcastHashJoin LeftSemi BuildRight (77) + : : :- Filter (72) + : : : +- Scan parquet (71) + : : +- BroadcastExchange (76) + : : +- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Filter (90) + : +- HashAggregate (89) + : +- Exchange (88) + : +- HashAggregate (87) + : +- BroadcastHashJoin LeftSemi BuildRight (86) + : :- Project (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (85) + : +- Project (84) + : +- Filter (83) + : +- Scan parquet (82) + +- BroadcastExchange (99) + +- Project (98) + +- Filter (97) + +- Scan parquet (96) (1) Scan parquet @@ -105,442 +107,450 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) AQEShuffleRead +(7) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) Scan parquet +(11) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(12) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(14) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(23) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(24) ProjectExecTransformer +(27) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(28) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(25) ReusedExchange [Reuses operator id: 13] +(29) ReusedExchange [Reuses operator id: 16] Output [1]: [p_partkey#X] -(26) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(27) InputAdapter +(31) InputAdapter Input [1]: [p_partkey#X] -(28) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [p_partkey#X] -(29) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(31) ProjectExecTransformer +(35) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(33) ColumnarExchange +(37) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(38) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(35) InputAdapter +(39) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) InputIteratorTransformer +(40) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(38) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(39) FilterExecTransformer +(43) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(40) BroadcastHashJoinExecTransformer +(44) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(41) ProjectExecTransformer -Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] -Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] - -(42) WholeStageCodegenTransformer (X) -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: false - -(43) ColumnarExchange -Input [2]: [hash_partition_key#X, ps_suppkey#X] -Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] - -(44) ShuffleQueryStage +(45) ProjectExecTransformer Output [1]: [ps_suppkey#X] -Arguments: X +Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(45) AQEShuffleRead +(46) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] -Arguments: local +Arguments: false -(46) ColumnarBroadcastExchange +(47) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastQueryStage +(48) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(48) InputAdapter +(49) InputAdapter Input [1]: [ps_suppkey#X] -(49) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(50) BroadcastHashJoinExecTransformer +(51) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(51) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(52) Scan parquet +(53) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(53) ProjectExecTransformer +(54) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(55) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(54) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(55) ColumnarBroadcastExchange +(57) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastQueryStage +(58) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(57) InputAdapter +(59) InputAdapter Input [1]: [n_nationkey#X] -(58) InputIteratorTransformer +(60) InputIteratorTransformer Input [1]: [n_nationkey#X] -(59) BroadcastHashJoinExecTransformer +(61) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(60) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(62) ColumnarExchange +(64) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(63) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(64) AQEShuffleRead +(66) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(66) Scan parquet +(68) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(68) Exchange +(70) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(71) Scan parquet +(73) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(72) Filter +(74) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(73) Project +(75) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(74) BroadcastExchange +(76) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(76) BroadcastExchange +(78) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(77) Scan parquet +(79) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(78) Filter +(80) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(79) Project +(81) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(80) Scan parquet +(82) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(81) Filter +(83) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(82) Project +(84) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(83) BroadcastExchange +(85) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(84) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(85) HashAggregate +(87) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(86) Exchange +(88) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) HashAggregate +(89) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(88) Filter +(90) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(90) Project +(92) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(91) Exchange +(93) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(93) Project +(95) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(94) Scan parquet +(96) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(95) Filter +(97) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(96) Project +(98) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(97) BroadcastExchange +(99) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(98) BroadcastHashJoin +(100) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(99) Project +(101) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(100) Exchange +(102) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Sort +(103) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(102) AdaptiveSparkPlan +(104) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index f8a97759d567..ac047fdbcabf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -1,85 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (91) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ InputIteratorTransformer (6) - : : : +- ^ InputAdapter (5) - : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : +- ColumnarBroadcastExchange (3) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) - : : : :- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ BroadcastQueryStage (12), Statistics(X) - : : : +- ColumnarBroadcastExchange (11) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20), Statistics(X) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (75) - : +- BroadcastHashJoin Inner BuildRight (74) - : :- Project (69) - : : +- BroadcastHashJoin Inner BuildLeft (68) - : : :- BroadcastExchange (56) - : : : +- Filter (55) - : : : +- Scan parquet (54) - : : +- BroadcastHashJoin LeftAnti BuildRight (67) - : : :- BroadcastHashJoin LeftSemi BuildRight (62) - : : : :- Project (59) - : : : : +- Filter (58) - : : : : +- Scan parquet (57) - : : : +- BroadcastExchange (61) - : : : +- Scan parquet (60) - : : +- BroadcastExchange (66) - : : +- Project (65) - : : +- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (73) - : +- Project (72) - : +- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (79) - +- Project (78) - +- Filter (77) - +- Scan parquet (76) + TakeOrderedAndProject (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (80) + : +- BroadcastHashJoin Inner BuildRight (79) + : :- Project (74) + : : +- BroadcastHashJoin Inner BuildLeft (73) + : : :- BroadcastExchange (61) + : : : +- Filter (60) + : : : +- Scan parquet (59) + : : +- BroadcastHashJoin LeftAnti BuildRight (72) + : : :- BroadcastHashJoin LeftSemi BuildRight (67) + : : : :- Project (64) + : : : : +- Filter (63) + : : : : +- Scan parquet (62) + : : : +- BroadcastExchange (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (71) + : : +- Project (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (84) + +- Project (83) + +- Filter (82) + +- Scan parquet (81) (1) Scan parquet @@ -89,382 +94,402 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(9) Scan parquet +(11) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(12) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(16) Scan parquet +(18) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(17) ProjectExecTransformer +(19) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(20) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(23) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(24) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(25) ProjectExecTransformer +(28) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(26) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(27) ProjectExecTransformer +(30) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(31) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(28) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastQueryStage +(34) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [1]: [o_orderkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [1]: [o_orderkey#X] -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(36) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(37) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(44) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [1]: [n_nationkey#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [1]: [n_nationkey#X] -(42) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(43) ProjectExecTransformer +(48) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(44) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(45) ProjectExecTransformer +(50) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(46) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(47) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(49) InputAdapter +(54) InputAdapter Input [2]: [s_name#X, count#X] -(50) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(51) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(53) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(54) Scan parquet +(59) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) BroadcastExchange +(61) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(62) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(63) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(59) Project +(64) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(60) Scan parquet +(65) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) BroadcastExchange +(66) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(63) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(64) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(65) Project +(70) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) BroadcastExchange +(71) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(69) Project +(74) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(70) Scan parquet +(75) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(76) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(72) Project +(77) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(73) BroadcastExchange +(78) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(74) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(75) Project +(80) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(76) Scan parquet +(81) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(77) Filter +(82) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(78) Project +(83) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(79) BroadcastExchange +(84) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(81) Project +(86) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(82) HashAggregate +(87) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(83) Exchange +(88) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(89) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(85) TakeOrderedAndProject +(90) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(86) AdaptiveSparkPlan +(91) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index 64084ebda520..3306ea6b2fd9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,39 +1,40 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (38) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ SortExecTransformer (23) - +- ^ InputIteratorTransformer (22) - +- ^ InputAdapter (21) - +- ^ ShuffleQueryStage (20), Statistics(X) - +- ColumnarExchange (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (7) - +- ^ InputAdapter (6) - +- ^ BroadcastQueryStage (5), Statistics(X) - +- ColumnarBroadcastExchange (4) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (36) - +- Exchange (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin LeftAnti BuildRight (30) - :- Filter (27) - : +- Scan parquet (26) - +- BroadcastExchange (29) - +- Scan parquet (28) + Sort (37) + +- Exchange (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin LeftAnti BuildRight (31) + :- Filter (28) + : +- Scan parquet (27) + +- BroadcastExchange (30) + +- Scan parquet (29) (1) Scan parquet @@ -43,270 +44,279 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [o_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [o_custkey#X] -(8) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(10) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(14) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(19) ColumnarExchange +(20) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) InputAdapter +(22) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(22) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) SortExecTransformer +(24) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(25) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(26) Scan parquet +(27) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(27) Filter +(28) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(28) Scan parquet +(29) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(29) BroadcastExchange +(30) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(31) Project +(32) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(32) HashAggregate +(33) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(35) Exchange +(36) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(37) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(37) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (48) - +- ^ RegularHashAggregateExecTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ShuffleQueryStage (43), Statistics(X) - +- ColumnarExchange (42) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ Scan parquet (38) + VeloxColumnarToRowExec (50) + +- ^ RegularHashAggregateExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) -(38) Scan parquet +(39) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(39) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(41) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(40) FlushableHashAggregateExecTransformer +(42) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(41) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(42) ColumnarExchange +(44) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(44) InputAdapter +(46) InputAdapter Input [2]: [sum#X, count#X] -(45) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [sum#X, count#X] -(46) RegularHashAggregateExecTransformer +(48) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(48) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(49) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(50) Filter +(52) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(51) Project +(53) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(52) HashAggregate +(54) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(53) Exchange +(55) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index 8b97940da5b5..0bcfb2c8228d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,52 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (31) - +- TakeOrderedAndProjectExecTransformer (30) - +- ^ ProjectExecTransformer (28) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ ProjectExecTransformer (10) - : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ BroadcastQueryStage (15), Statistics(X) - +- ColumnarBroadcastExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ Scan parquet (11) + VeloxColumnarToRowExec (34) + +- TakeOrderedAndProjectExecTransformer (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Project (45) - +- BroadcastHashJoin Inner BuildRight (44) - :- Project (39) - : +- BroadcastHashJoin Inner BuildLeft (38) - : :- BroadcastExchange (35) - : : +- Project (34) - : : +- Filter (33) - : : +- Scan parquet (32) - : +- Filter (37) - : +- Scan parquet (36) - +- BroadcastExchange (43) - +- Project (42) - +- Filter (41) - +- Scan parquet (40) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -56,222 +59,234 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(24) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(26) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(28) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(30) TakeOrderedAndProjectExecTransformer +(33) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(31) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(32) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(33) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(34) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(35) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(36) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(38) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(41) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(42) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(45) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(46) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(47) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(49) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 31cba5c73b0a..b547b4051bcf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- ^ InputAdapter (10) + +- ^ BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin LeftSemi BuildRight (35) - :- Project (30) - : +- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin LeftSemi BuildRight (37) + :- Project (32) + : +- Filter (31) + : +- Scan parquet (30) + +- BroadcastExchange (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,182 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) Scan parquet +(4) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(4) ProjectExecTransformer +(5) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(6) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(5) WholeStageCodegenTransformer (X) +(7) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(8) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(9) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(8) InputAdapter +(10) InputAdapter Input [1]: [l_orderkey#X] -(9) InputIteratorTransformer +(11) InputIteratorTransformer Input [1]: [l_orderkey#X] -(10) BroadcastHashJoinExecTransformer +(12) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(12) FlushableHashAggregateExecTransformer +(14) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(13) ProjectExecTransformer +(15) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(14) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(15) ColumnarExchange +(17) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(28) Scan parquet +(30) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(29) Filter +(31) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(30) Project +(32) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(31) Scan parquet +(33) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(32) Filter +(34) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(33) Project +(35) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(34) BroadcastExchange +(36) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(37) HashAggregate +(39) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(38) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index dfdfede045a2..a87ef7c618c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,92 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ SortExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (62) - : : : : : +- Filter (61) - : : : : : +- Scan parquet (60) - : : : : +- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Project (71) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -96,414 +102,438 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(10) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(15) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(24) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(27) Scan parquet +(31) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(33) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(36) ProjectExecTransformer +(41) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(42) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(37) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(44) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(45) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(40) InputAdapter +(46) InputAdapter Input [1]: [r_regionkey#X] -(41) InputIteratorTransformer +(47) InputIteratorTransformer Input [1]: [r_regionkey#X] -(42) BroadcastHashJoinExecTransformer +(48) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(43) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(44) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(45) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(46) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(47) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(55) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(52) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(53) ColumnarExchange +(59) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(55) InputAdapter +(61) InputAdapter Input [2]: [n_name#X, revenue#X] -(56) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(57) SortExecTransformer +(63) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(60) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(62) BroadcastExchange +(68) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(65) Project +(71) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(66) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(67) Project +(73) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(70) BroadcastExchange +(76) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(78) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(79) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(80) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(81) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(77) Project +(83) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(84) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(80) BroadcastExchange +(86) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(83) Scan parquet +(89) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(84) Filter +(90) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(85) Project +(91) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(86) BroadcastExchange +(92) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(88) Project +(94) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) HashAggregate +(95) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) Exchange +(96) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(97) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(92) Exchange +(98) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(99) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index 2e51f22a6b0e..51c5836bdd11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 0a8c1111c851..37e9e5214256 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,87 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (33) - : +- ^ BroadcastHashJoinExecTransformer Inner (32) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ ProjectExecTransformer (17) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) - : : : :- ^ ProjectExecTransformer (9) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (15) - : : : +- ^ InputAdapter (14) - : : : +- ^ BroadcastQueryStage (13), Statistics(X) - : : : +- ColumnarBroadcastExchange (12) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ BroadcastQueryStage (21), Statistics(X) - : : +- ColumnarBroadcastExchange (20) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ BroadcastQueryStage (29), Statistics(X) - : +- ColumnarBroadcastExchange (28) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ReusedExchange (34) + VeloxColumnarToRowExec (60) + +- ^ SortExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- ^ InputAdapter (17) + : : : +- ^ BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- ^ InputAdapter (26) + : : +- ^ BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (87) - +- Exchange (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (77) - : +- BroadcastHashJoin Inner BuildRight (76) - : :- Project (72) - : : +- BroadcastHashJoin Inner BuildRight (71) - : : :- Project (67) - : : : +- BroadcastHashJoin Inner BuildRight (66) - : : : :- Project (62) - : : : : +- BroadcastHashJoin Inner BuildLeft (61) - : : : : :- BroadcastExchange (58) - : : : : : +- Filter (57) - : : : : : +- Scan parquet (56) - : : : : +- Filter (60) - : : : : +- Scan parquet (59) - : : : +- BroadcastExchange (65) - : : : +- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (75) - : +- Filter (74) - : +- Scan parquet (73) - +- BroadcastExchange (80) - +- Filter (79) - +- Scan parquet (78) + Sort (92) + +- Exchange (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (63) + : : : : : +- Filter (62) + : : : : : +- Scan parquet (61) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -91,386 +96,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) BroadcastHashJoinExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(13) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(14) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) BroadcastHashJoinExecTransformer +(19) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(17) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(18) Scan parquet +(21) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(19) WholeStageCodegenTransformer (X) +(22) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(23) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(24) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(21) BroadcastQueryStage +(25) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(22) InputAdapter +(26) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(23) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(24) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(25) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(26) Scan parquet +(30) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(27) WholeStageCodegenTransformer (X) +(31) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(32) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(28) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastQueryStage +(34) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(30) InputAdapter +(35) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(31) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(32) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(38) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(34) ReusedExchange [Reuses operator id: 28] +(39) ReusedExchange [Reuses operator id: 33] Output [2]: [n_nationkey#X, n_name#X] -(35) BroadcastQueryStage +(40) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(36) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(37) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(38) BroadcastHashJoinExecTransformer +(43) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(39) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(40) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(48) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(49) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(50) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(51) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(48) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(49) ColumnarExchange +(54) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(55) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(51) InputAdapter +(56) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(52) InputIteratorTransformer +(57) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(53) SortExecTransformer +(58) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(54) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(56) Scan parquet +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(58) BroadcastExchange +(63) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(59) Scan parquet +(64) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(61) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(62) Project +(67) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(63) Scan parquet +(68) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(64) Filter +(69) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(65) BroadcastExchange +(70) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(66) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(67) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(73) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(70) BroadcastExchange +(75) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(72) Project +(77) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(73) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(74) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(75) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(77) Project +(82) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(78) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(79) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(82) Project +(87) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(88) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(86) Exchange +(91) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Sort +(92) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 6822e2becd51..9b490b9ba436 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,117 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ SortExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ShuffleQueryStage (64), Statistics(X) - +- ColumnarExchange (63) - +- ^ ProjectExecTransformer (61) - +- ^ FlushableHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ BroadcastHashJoinExecTransformer Inner (58) - :- ^ ProjectExecTransformer (50) - : +- ^ BroadcastHashJoinExecTransformer Inner (49) - : :- ^ ProjectExecTransformer (42) - : : +- ^ BroadcastHashJoinExecTransformer Inner (41) - : : :- ^ ProjectExecTransformer (34) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : : : :- ^ ProjectExecTransformer (26) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : : : :- ^ ProjectExecTransformer (18) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : : : :- ^ ProjectExecTransformer (10) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (16) - : : : : : +- ^ InputAdapter (15) - : : : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (13) - : : : : : +- ^ Scan parquet (11) - : : : : +- ^ InputIteratorTransformer (24) - : : : : +- ^ InputAdapter (23) - : : : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : : : +- ColumnarBroadcastExchange (21) - : : : : +- ^ Scan parquet (19) - : : : +- ^ InputIteratorTransformer (32) - : : : +- ^ InputAdapter (31) - : : : +- ^ BroadcastQueryStage (30), Statistics(X) - : : : +- ColumnarBroadcastExchange (29) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (40) - : : +- ^ InputAdapter (39) - : : +- ^ BroadcastQueryStage (38), Statistics(X) - : : +- ColumnarBroadcastExchange (37) - : : +- ^ Scan parquet (35) - : +- ^ InputIteratorTransformer (48) - : +- ^ InputAdapter (47) - : +- ^ BroadcastQueryStage (46), Statistics(X) - : +- ColumnarBroadcastExchange (45) - : +- ^ Scan parquet (43) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ BroadcastQueryStage (55), Statistics(X) - +- ColumnarBroadcastExchange (54) - +- ^ ProjectExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (84) + +- ^ SortExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- ^ InputAdapter (18) + : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- ^ InputAdapter (27) + : : : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- ^ InputAdapter (36) + : : : +- ^ BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- ^ InputAdapter (54) + : +- ^ BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- BroadcastHashJoin Inner BuildRight (114) - :- Project (109) - : +- BroadcastHashJoin Inner BuildRight (108) - : :- Project (104) - : : +- BroadcastHashJoin Inner BuildRight (103) - : : :- Project (99) - : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : :- Project (94) - : : : : +- BroadcastHashJoin Inner BuildRight (93) - : : : : :- Project (89) - : : : : : +- BroadcastHashJoin Inner BuildRight (88) - : : : : : :- Project (84) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) - : : : : : : :- BroadcastExchange (80) - : : : : : : : +- Project (79) - : : : : : : : +- Filter (78) - : : : : : : : +- Scan parquet (77) - : : : : : : +- Filter (82) - : : : : : : +- Scan parquet (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- Filter (86) - : : : : : +- Scan parquet (85) - : : : : +- BroadcastExchange (92) - : : : : +- Filter (91) - : : : : +- Scan parquet (90) - : : : +- BroadcastExchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- BroadcastExchange (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- BroadcastExchange (107) - : +- Filter (106) - : +- Scan parquet (105) - +- BroadcastExchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (128) + +- Exchange (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- BroadcastHashJoin Inner BuildRight (122) + :- Project (117) + : +- BroadcastHashJoin Inner BuildRight (116) + : :- Project (112) + : : +- BroadcastHashJoin Inner BuildRight (111) + : : :- Project (107) + : : : +- BroadcastHashJoin Inner BuildRight (106) + : : : :- Project (102) + : : : : +- BroadcastHashJoin Inner BuildRight (101) + : : : : :- Project (97) + : : : : : +- BroadcastHashJoin Inner BuildRight (96) + : : : : : :- Project (92) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) + : : : : : : :- BroadcastExchange (88) + : : : : : : : +- Project (87) + : : : : : : : +- Filter (86) + : : : : : : : +- Scan parquet (85) + : : : : : : +- Filter (90) + : : : : : : +- Scan parquet (89) + : : : : : +- BroadcastExchange (95) + : : : : : +- Filter (94) + : : : : : +- Scan parquet (93) + : : : : +- BroadcastExchange (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- BroadcastExchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- BroadcastExchange (110) + : : +- Filter (109) + : : +- Scan parquet (108) + : +- BroadcastExchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- BroadcastExchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,534 +129,566 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(43) Scan parquet +(49) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(44) WholeStageCodegenTransformer (X) +(50) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(51) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(45) ColumnarBroadcastExchange +(52) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastQueryStage +(53) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(47) InputAdapter +(54) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(48) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastHashJoinExecTransformer +(56) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(50) ProjectExecTransformer +(57) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(51) Scan parquet +(58) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(52) ProjectExecTransformer +(59) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(60) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(53) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(54) ColumnarBroadcastExchange +(62) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(55) BroadcastQueryStage +(63) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(56) InputAdapter +(64) InputAdapter Input [1]: [r_regionkey#X] -(57) InputIteratorTransformer +(65) InputIteratorTransformer Input [1]: [r_regionkey#X] -(58) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(59) ProjectExecTransformer +(67) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(60) FlushableHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(61) ProjectExecTransformer +(69) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(62) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(63) ColumnarExchange +(71) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(64) ShuffleQueryStage +(72) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(65) InputAdapter +(73) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(66) InputIteratorTransformer +(74) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(67) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(68) ProjectExecTransformer +(76) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(69) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(70) ColumnarExchange +(78) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(72) InputAdapter +(80) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(73) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(74) SortExecTransformer +(82) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(75) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(76) VeloxColumnarToRowExec +(84) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(77) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(78) Filter +(86) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(79) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(80) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(90) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(84) Project +(92) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(85) Scan parquet +(93) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(94) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(95) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(97) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(91) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(92) BroadcastExchange +(100) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(93) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(94) Project +(102) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Scan parquet +(103) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(104) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) BroadcastExchange +(105) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(98) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(99) Project +(107) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(100) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(101) Filter +(109) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(102) BroadcastExchange +(110) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(103) BroadcastHashJoin +(111) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(112) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(105) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(106) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(107) BroadcastExchange +(115) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(108) BroadcastHashJoin +(116) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) Project +(117) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(110) Scan parquet +(118) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(111) Filter +(119) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(112) Project +(120) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(113) BroadcastExchange +(121) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(114) BroadcastHashJoin +(122) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(115) Project +(123) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(116) HashAggregate +(124) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(117) Exchange +(125) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(126) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(119) Exchange +(127) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(128) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index bec2b05cc2a3..8523cbeff2ab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,90 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ SortExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (64) + +- ^ SortExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ^ InputAdapter (60) + +- ^ ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (91) - +- Exchange (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildRight (75) - : : :- Project (71) - : : : +- BroadcastHashJoin Inner BuildRight (70) - : : : :- Project (66) - : : : : +- BroadcastHashJoin Inner BuildLeft (65) - : : : : :- BroadcastExchange (62) - : : : : : +- Project (61) - : : : : : +- Filter (60) - : : : : : +- Scan parquet (59) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (84) - +- Filter (83) - +- Scan parquet (82) + Sort (97) + +- Exchange (96) + +- HashAggregate (95) + +- Exchange (94) + +- HashAggregate (93) + +- Project (92) + +- BroadcastHashJoin Inner BuildRight (91) + :- Project (87) + : +- BroadcastHashJoin Inner BuildRight (86) + : :- Project (82) + : : +- BroadcastHashJoin Inner BuildRight (81) + : : :- Project (77) + : : : +- BroadcastHashJoin Inner BuildRight (76) + : : : :- Project (72) + : : : : +- BroadcastHashJoin Inner BuildLeft (71) + : : : : :- BroadcastExchange (68) + : : : : : +- Project (67) + : : : : : +- Filter (66) + : : : : : +- Scan parquet (65) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (75) + : : : +- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- BroadcastExchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -94,406 +100,430 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(43) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(52) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(56) SortExecTransformer +(62) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(59) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(60) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(61) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(62) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(66) Project +(72) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(67) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(68) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(69) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(71) Project +(77) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(72) Scan parquet +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(73) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(74) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(75) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(76) Project +(82) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(77) Scan parquet +(83) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(84) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(79) BroadcastExchange +(85) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(87) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(82) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(84) BroadcastExchange +(90) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(85) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(92) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(87) HashAggregate +(93) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(88) Exchange +(94) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(95) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(90) Exchange +(96) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Sort +(97) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(92) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index 0217849e5de2..41613c36f7bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index fc3503c42fef..276a885015d0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,64 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ FlushableHashAggregateExecTransformer (28) - +- ^ ProjectExecTransformer (27) - +- ^ BroadcastHashJoinExecTransformer Inner (26) - :- ^ ProjectExecTransformer (19) - : +- ^ BroadcastHashJoinExecTransformer Inner (18) - : :- ^ ProjectExecTransformer (10) - : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : :- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ Scan parquet (2) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ BroadcastQueryStage (15), Statistics(X) - : +- ColumnarBroadcastExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ Scan parquet (11) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ BroadcastQueryStage (23), Statistics(X) - +- ColumnarBroadcastExchange (22) - +- ^ Scan parquet (20) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- BroadcastHashJoin Inner BuildRight (57) - :- Project (53) - : +- BroadcastHashJoin Inner BuildRight (52) - : :- Project (47) - : : +- BroadcastHashJoin Inner BuildRight (46) - : : :- Filter (41) - : : : +- Scan parquet (40) - : : +- BroadcastExchange (45) - : : +- Project (44) - : : +- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (51) - : +- Project (50) - : +- Filter (49) - : +- Scan parquet (48) - +- BroadcastExchange (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -68,286 +72,302 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(3) ProjectExecTransformer +(4) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(5) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(4) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) Scan parquet +(23) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(21) WholeStageCodegenTransformer (X) +(24) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(25) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(22) ColumnarBroadcastExchange +(26) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(27) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(24) InputAdapter +(28) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(25) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(26) BroadcastHashJoinExecTransformer +(30) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(27) ProjectExecTransformer +(31) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(28) FlushableHashAggregateExecTransformer +(32) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(29) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(30) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(31) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(40) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(42) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(43) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(44) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(46) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(47) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(48) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(49) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(50) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(51) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(52) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(53) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(54) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(55) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(56) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(58) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(59) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(60) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index 943d59c56aaf..f8d59a67a22b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ FilterExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ BroadcastHashJoinExecTransformer Inner (17) - :- ^ ProjectExecTransformer (9) - : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ BroadcastQueryStage (14), Statistics(X) - +- ColumnarBroadcastExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ^ InputAdapter (34) + +- ^ ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- ^ InputAdapter (8) + : +- ^ BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (54) - +- Exchange (53) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildRight (41) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- BroadcastHashJoin Inner BuildRight (50) + :- Project (45) + : +- BroadcastHashJoin Inner BuildRight (44) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (49) + +- Project (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -60,472 +63,489 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(5) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(11) ProjectExecTransformer +(13) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(14) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(12) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [1]: [n_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [1]: [n_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(19) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(20) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(21) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(22) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(29) ColumnarExchange +(32) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [2]: [ps_partkey#X, value#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(36) Scan parquet +(39) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(40) BroadcastExchange +(43) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(41) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(42) Project +(45) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(45) Project +(48) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(46) BroadcastExchange +(49) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) Project +(51) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(49) HashAggregate +(52) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(50) Exchange +(53) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(52) Filter +(55) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(53) Exchange +(56) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Sort +(57) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(55) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 27 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (95) +Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ FlushableHashAggregateExecTransformer (69) - +- ^ ProjectExecTransformer (68) - +- ^ BroadcastHashJoinExecTransformer Inner (67) - :- ^ ProjectExecTransformer (62) - : +- ^ BroadcastHashJoinExecTransformer Inner (61) - : :- ^ Scan parquet (56) - : +- ^ InputIteratorTransformer (60) - : +- ^ InputAdapter (59) - : +- ^ BroadcastQueryStage (58), Statistics(X) - : +- ReusedExchange (57) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ BroadcastQueryStage (64), Statistics(X) - +- ReusedExchange (63) + VeloxColumnarToRowExec (82) + +- ^ ProjectExecTransformer (80) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : :- ^ NoopFilter (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == - HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Project (91) - +- BroadcastHashJoin Inner BuildRight (90) - :- Project (85) - : +- BroadcastHashJoin Inner BuildRight (84) - : :- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (83) - : +- Filter (82) - : +- Scan parquet (81) - +- BroadcastExchange (89) - +- Project (88) - +- Filter (87) - +- Scan parquet (86) - - -(56) Scan parquet + HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Filter (84) + : : +- Scan parquet (83) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) + + +(59) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(57) ReusedExchange [Reuses operator id: 4] +(60) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(61) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(58) BroadcastQueryStage +(62) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(59) InputAdapter +(63) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(60) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(65) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(62) ProjectExecTransformer +(66) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(63) ReusedExchange [Reuses operator id: 13] +(67) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(64) BroadcastQueryStage +(68) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(65) InputAdapter +(69) InputAdapter Input [1]: [n_nationkey#X] -(66) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [n_nationkey#X] -(67) BroadcastHashJoinExecTransformer +(71) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(68) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(69) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(75) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(77) InputAdapter Input [2]: [sum#X, isEmpty#X] -(74) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(76) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(77) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(78) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(79) Scan parquet +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) Filter +(84) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(81) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(82) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(83) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(84) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(85) Project +(89) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(86) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(87) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(88) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(89) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(90) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(91) Project +(95) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(92) HashAggregate +(96) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(93) Exchange +(97) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(98) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(95) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index f53f974558bd..9e142ea961f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ^ InputAdapter (18) + +- ^ ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- BroadcastHashJoin Inner BuildLeft (33) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -46,176 +48,184 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(12) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(13) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(14) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(16) InputAdapter +(18) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(17) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(32) Project +(34) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(33) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(34) Project +(36) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(36) Exchange +(38) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index 170698c552ac..9841ce1395e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ ProjectExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- BroadcastHashJoin LeftOuter BuildRight (41) - :- Scan parquet (36) - +- BroadcastExchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- BroadcastHashJoin LeftOuter BuildRight (42) + :- Scan parquet (37) + +- BroadcastExchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -62,222 +63,226 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) ProjectExecTransformer +(3) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(4) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(7) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(11) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(14) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [2]: [c_custkey#X, count#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(39) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(40) BroadcastExchange +(41) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(42) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(43) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(44) Exchange +(45) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index b4a96c56cfb8..fe13deb40500 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ ProjectExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (22) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (31) - +- Exchange (30) - +- HashAggregate (29) - +- Project (28) - +- BroadcastHashJoin Inner BuildRight (27) - :- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- BroadcastExchange (26) - +- Filter (25) - +- Scan parquet (24) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -39,146 +41,154 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) Scan parquet +(4) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(6) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(21) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(22) Filter +(24) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(23) Project +(25) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(26) BroadcastExchange +(28) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(28) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(29) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(30) Exchange +(32) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(31) HashAggregate +(33) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 75af0c33004f..4b761198f3e3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (41) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- AQEShuffleRead (23) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ InputIteratorTransformer (6) - : +- ^ InputAdapter (5) - : +- ^ BroadcastQueryStage (4), Statistics(X) - : +- ColumnarBroadcastExchange (3) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (17) - +- ^ RegularHashAggregateExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FlushableHashAggregateExecTransformer (9) - +- ^ ProjectExecTransformer (8) - +- ^ Scan parquet (7) + VeloxColumnarToRowExec (26) + +- AQEShuffleRead (25) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner (20) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (38) - +- Exchange (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Filter (34) - +- HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- Filter (29) - +- Scan parquet (28) + Sort (40) + +- Exchange (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Filter (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -46,326 +48,339 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(18) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(23) AQEShuffleRead +(25) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(24) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) Scan parquet +(27) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(27) BroadcastExchange +(29) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(30) Project +(32) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(32) Exchange +(34) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(34) Filter +(36) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(36) Project +(38) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(37) Exchange +(39) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Sort +(40) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(39) AdaptiveSparkPlan +(41) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 17 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ ProjectExecTransformer (50) - +- ^ RegularHashAggregateExecTransformer (49) - +- ^ InputIteratorTransformer (48) - +- ^ InputAdapter (47) - +- ^ ShuffleQueryStage (46), Statistics(X) - +- ColumnarExchange (45) - +- ^ ProjectExecTransformer (43) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (57) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ NoopFilter (43) + +- ^ Scan parquet (42) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Exchange (59) - +- HashAggregate (58) - +- Project (57) - +- Filter (56) - +- Scan parquet (55) + HashAggregate (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- Filter (59) + +- Scan parquet (58) -(40) Scan parquet +(42) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(44) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(42) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(43) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(45) ColumnarExchange +(48) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(46) ShuffleQueryStage +(49) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(47) InputAdapter +(50) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) InputIteratorTransformer +(51) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(50) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(53) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(54) VeloxColumnarToRowExec +(57) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(55) Scan parquet +(58) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(56) Filter +(59) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(57) Project +(60) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(58) HashAggregate +(61) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(59) Exchange +(62) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) HashAggregate +(63) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(61) HashAggregate +(64) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(63) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index f87640ad9456..f178803b43e9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,62 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (41) - +- ^ SortExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ ProjectExecTransformer (27) - +- ^ FlushableHashAggregateExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ BroadcastHashJoinExecTransformer Inner (16) - :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ BroadcastQueryStage (6), Statistics(X) - : +- ColumnarBroadcastExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ Scan parquet (2) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ BroadcastQueryStage (13), Statistics(X) - +- ColumnarBroadcastExchange (12) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- BroadcastHashJoin Inner BuildRight (52) - :- BroadcastHashJoin LeftAnti BuildRight (48) - : :- Filter (43) - : : +- Scan parquet (42) - : +- BroadcastExchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- BroadcastExchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- BroadcastHashJoin Inner BuildRight (46) + :- BroadcastHashJoin LeftAnti BuildRight (42) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -66,282 +61,255 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) Scan parquet -Output [2]: [s_suppkey#X, s_comment#X] -Batched: true -Location: InMemoryFileIndex [*] -PushedFilters: [IsNotNull(s_comment)] -ReadSchema: struct - -(3) ProjectExecTransformer -Output [1]: [s_suppkey#X] -Input [2]: [s_suppkey#X, s_comment#X] - -(4) WholeStageCodegenTransformer (X) -Input [1]: [s_suppkey#X] -Arguments: false - -(5) ColumnarBroadcastExchange -Input [1]: [s_suppkey#X] -Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] - -(6) BroadcastQueryStage -Output [1]: [s_suppkey#X] -Arguments: X - -(7) InputAdapter -Input [1]: [s_suppkey#X] - -(8) InputIteratorTransformer -Input [1]: [s_suppkey#X] - -(9) BroadcastHashJoinExecTransformer -Left keys [1]: [ps_suppkey#X] -Right keys [1]: [s_suppkey#X] -Join type: LeftAnti -Join condition: None +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] -(10) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(4) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(5) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(12) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(7) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(14) InputAdapter +(8) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(17) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) ProjectExecTransformer +(13) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(21) ColumnarExchange +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(23) InputAdapter +(17) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(29) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(31) InputAdapter +(25) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(34) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) ColumnarExchange +(29) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(37) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) SortExecTransformer +(33) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(40) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(41) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) Scan parquet +(36) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(43) Filter +(37) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(44) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(45) Filter +(39) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(46) Project +(40) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(47) BroadcastExchange +(41) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(48) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(49) Scan parquet +(43) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(44) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) BroadcastExchange +(45) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(52) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(53) Project +(47) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(48) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(49) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(50) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(54) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(55) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index 9c4a028c442c..2484955032d0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,83 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- TakeOrderedAndProjectExecTransformer (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (26) - : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : :- ^ InputIteratorTransformer (6) - : : +- ^ InputAdapter (5) - : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : +- ColumnarBroadcastExchange (3) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) - : :- ^ Scan parquet (7) - : +- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ BroadcastQueryStage (21), Statistics(X) - : +- ColumnarBroadcastExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ FilterExecTransformer (17) - : +- ^ RegularHashAggregateExecTransformer (16) - : +- ^ InputIteratorTransformer (15) - : +- ^ InputAdapter (14) - : +- ^ ShuffleQueryStage (13), Statistics(X) - : +- ColumnarExchange (12) - : +- ^ ProjectExecTransformer (10) - : +- ^ FlushableHashAggregateExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ColumnarBroadcastExchange (34) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) - :- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ BroadcastQueryStage (29), Statistics(X) - +- ReusedExchange (28) + VeloxColumnarToRowExec (53) + +- TakeOrderedAndProjectExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- BroadcastHashJoin Inner BuildRight (77) - :- Project (65) - : +- BroadcastHashJoin Inner BuildLeft (64) - : :- BroadcastExchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- BroadcastHashJoin LeftSemi BuildRight (63) - : :- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastExchange (62) - : +- Project (61) - : +- Filter (60) - : +- HashAggregate (59) - : +- Exchange (58) - : +- HashAggregate (57) - : +- Scan parquet (56) - +- BroadcastExchange (76) - +- BroadcastHashJoin LeftSemi BuildRight (75) - :- Filter (67) - : +- Scan parquet (66) - +- BroadcastExchange (74) - +- Project (73) - +- Filter (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Scan parquet (68) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -87,383 +90,395 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) Scan parquet +(9) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(10) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(11) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(12) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(16) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(17) FilterExecTransformer +(19) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [1]: [l_orderkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [l_orderkey#X] -(24) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(25) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(27) Scan parquet +(29) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(28) ReusedExchange [Reuses operator id: 20] +(30) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(31) ReusedExchange [Reuses operator id: 22] Output [1]: [l_orderkey#X] -(29) BroadcastQueryStage +(32) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [1]: [l_orderkey#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [1]: [l_orderkey#X] -(32) BroadcastHashJoinExecTransformer +(35) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(34) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(38) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(42) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(40) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(44) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(49) TakeOrderedAndProjectExecTransformer +(52) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(53) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(54) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(56) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(57) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(58) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(60) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(61) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(62) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(64) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(65) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(66) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(75) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(76) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(79) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(80) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(82) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(83) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index 16e791cab81b..1a2a13c21db8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -1,34 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (31) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer Inner (9) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (30) - +- Exchange (29) - +- HashAggregate (28) - +- Project (27) - +- BroadcastHashJoin Inner BuildRight (26) - :- Project (22) - : +- Filter (21) - : +- Scan parquet (20) - +- BroadcastExchange (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (32) + +- Exchange (31) + +- HashAggregate (30) + +- Project (29) + +- BroadcastHashJoin Inner BuildRight (28) + :- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- BroadcastExchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -38,142 +40,150 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) Scan parquet +(4) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(5) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(6) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(5) ColumnarBroadcastExchange +(7) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(6) BroadcastQueryStage +(8) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(7) InputAdapter +(9) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(8) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [revenue#X] -(20) Scan parquet +(22) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(21) Filter +(23) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(22) Project +(24) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(23) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(24) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(25) BroadcastExchange +(27) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastHashJoin +(28) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(27) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(28) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(29) Exchange +(31) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(30) HashAggregate +(32) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(31) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index 70067664658e..ed40d3bbe23d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,91 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (96) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- AQEShuffleRead (54) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ BroadcastHashJoinExecTransformer Inner (49) - :- ^ ProjectExecTransformer (41) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (40) - : :- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ BroadcastQueryStage (37), Statistics(X) - : +- ColumnarBroadcastExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (15) - : : +- ^ InputAdapter (14) - : : +- ^ BroadcastQueryStage (13), Statistics(X) - : : +- ColumnarBroadcastExchange (12) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - : : :- ^ Scan parquet (2) - : : +- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ BroadcastQueryStage (7), Statistics(X) - : : +- ColumnarBroadcastExchange (6) - : : +- ^ ProjectExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ FilterExecTransformer (32) - : +- ^ ProjectExecTransformer (31) - : +- ^ RegularHashAggregateExecTransformer (30) - : +- ^ InputIteratorTransformer (29) - : +- ^ InputAdapter (28) - : +- ^ ShuffleQueryStage (27), Statistics(X) - : +- ColumnarExchange (26) - : +- ^ ProjectExecTransformer (24) - : +- ^ FlushableHashAggregateExecTransformer (23) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (22) - : :- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ BroadcastQueryStage (19), Statistics(X) - : +- ReusedExchange (18) - +- ^ InputIteratorTransformer (48) - +- ^ InputAdapter (47) - +- ^ BroadcastQueryStage (46), Statistics(X) - +- ColumnarBroadcastExchange (45) - +- ^ ProjectExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (60) + +- AQEShuffleRead (59) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ BroadcastHashJoinExecTransformer Inner (54) + :- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (44) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ InputIteratorTransformer (18) + : : +- ^ InputAdapter (17) + : : +- ^ BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (13) + : : :- ^ NoopFilter (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- ^ InputAdapter (11) + : : +- ^ BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ NoopFilter (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ^ InputAdapter (32) + : +- ^ ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - Sort (90) - +- Exchange (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin LeftSemi BuildRight (81) - : :- Filter (57) - : : +- Scan parquet (56) - : +- BroadcastExchange (80) - : +- Project (79) - : +- BroadcastHashJoin Inner BuildLeft (78) - : :- BroadcastExchange (65) - : : +- BroadcastHashJoin LeftSemi BuildRight (64) - : : :- Filter (59) - : : : +- Scan parquet (58) - : : +- BroadcastExchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Filter (77) - : +- HashAggregate (76) - : +- Exchange (75) - : +- HashAggregate (74) - : +- BroadcastHashJoin LeftSemi BuildRight (73) - : :- Project (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- BroadcastExchange (72) - : +- Project (71) - : +- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (95) + +- Exchange (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (87) + : +- BroadcastHashJoin LeftSemi BuildRight (86) + : :- Filter (62) + : : +- Scan parquet (61) + : +- BroadcastExchange (85) + : +- Project (84) + : +- BroadcastHashJoin Inner BuildLeft (83) + : :- BroadcastExchange (70) + : : +- BroadcastHashJoin LeftSemi BuildRight (69) + : : :- Filter (64) + : : : +- Scan parquet (63) + : : +- BroadcastExchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Filter (82) + : +- HashAggregate (81) + : +- Exchange (80) + : +- HashAggregate (79) + : +- BroadcastHashJoin LeftSemi BuildRight (78) + : :- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- BroadcastExchange (77) + : +- Project (76) + : +- Filter (75) + : +- Scan parquet (74) + +- BroadcastExchange (91) + +- Project (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -95,410 +100,430 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(3) Scan parquet +(4) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(5) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(4) ProjectExecTransformer +(6) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(7) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(5) WholeStageCodegenTransformer (X) +(8) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(9) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(10) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(8) InputAdapter +(11) InputAdapter Input [1]: [p_partkey#X] -(9) InputIteratorTransformer +(12) InputIteratorTransformer Input [1]: [p_partkey#X] -(10) BroadcastHashJoinExecTransformer +(13) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(11) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) Scan parquet +(19) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(17) ProjectExecTransformer +(20) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(21) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(18) ReusedExchange [Reuses operator id: 6] +(22) ReusedExchange [Reuses operator id: 9] Output [1]: [p_partkey#X] -(19) BroadcastQueryStage +(23) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(24) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(25) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(23) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(24) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(25) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(32) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(31) ProjectExecTransformer +(35) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(32) FilterExecTransformer +(36) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(36) ColumnarBroadcastExchange +(40) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastQueryStage +(41) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [ps_suppkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(40) BroadcastHashJoinExecTransformer +(44) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(41) ProjectExecTransformer +(45) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(43) ProjectExecTransformer +(47) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(48) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(44) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(45) ColumnarBroadcastExchange +(50) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(46) BroadcastQueryStage +(51) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(47) InputAdapter +(52) InputAdapter Input [1]: [n_nationkey#X] -(48) InputIteratorTransformer +(53) InputIteratorTransformer Input [1]: [n_nationkey#X] -(49) BroadcastHashJoinExecTransformer +(54) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(50) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(51) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(52) ColumnarExchange +(57) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(54) AQEShuffleRead +(59) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(56) Scan parquet +(61) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(58) Scan parquet +(63) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(59) Filter +(64) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(60) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(62) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(63) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(64) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(65) BroadcastExchange +(70) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(66) Scan parquet +(71) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(67) Filter +(72) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(68) Project +(73) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(69) Scan parquet +(74) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(70) Filter +(75) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(71) Project +(76) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(72) BroadcastExchange +(77) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(74) HashAggregate +(79) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(75) Exchange +(80) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) HashAggregate +(81) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(77) Filter +(82) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(78) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(79) Project +(84) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(80) BroadcastExchange +(85) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(82) Project +(87) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(83) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(84) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(85) Project +(90) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(86) BroadcastExchange +(91) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(93) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(89) Exchange +(94) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Sort +(95) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(91) AdaptiveSparkPlan +(96) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 80e1d1dc5f36..d0b14137ca5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -1,85 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (91) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ InputIteratorTransformer (6) - : : : +- ^ InputAdapter (5) - : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : +- ColumnarBroadcastExchange (3) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) - : : : :- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ BroadcastQueryStage (12), Statistics(X) - : : : +- ColumnarBroadcastExchange (11) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (22) - : : +- ^ InputAdapter (21) - : : +- ^ BroadcastQueryStage (20), Statistics(X) - : : +- ColumnarBroadcastExchange (19) - : : +- ^ ProjectExecTransformer (17) - : : +- ^ Scan parquet (16) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (75) - : +- BroadcastHashJoin Inner BuildRight (74) - : :- Project (69) - : : +- BroadcastHashJoin Inner BuildLeft (68) - : : :- BroadcastExchange (56) - : : : +- Filter (55) - : : : +- Scan parquet (54) - : : +- BroadcastHashJoin LeftAnti BuildRight (67) - : : :- BroadcastHashJoin LeftSemi BuildRight (62) - : : : :- Project (59) - : : : : +- Filter (58) - : : : : +- Scan parquet (57) - : : : +- BroadcastExchange (61) - : : : +- Scan parquet (60) - : : +- BroadcastExchange (66) - : : +- Project (65) - : : +- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (73) - : +- Project (72) - : +- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (79) - +- Project (78) - +- Filter (77) - +- Scan parquet (76) + TakeOrderedAndProject (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (80) + : +- BroadcastHashJoin Inner BuildRight (79) + : :- Project (74) + : : +- BroadcastHashJoin Inner BuildLeft (73) + : : :- BroadcastExchange (61) + : : : +- Filter (60) + : : : +- Scan parquet (59) + : : +- BroadcastHashJoin LeftAnti BuildRight (72) + : : :- BroadcastHashJoin LeftSemi BuildRight (67) + : : : :- Project (64) + : : : : +- Filter (63) + : : : : +- Scan parquet (62) + : : : +- BroadcastExchange (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (71) + : : +- Project (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (84) + +- Project (83) + +- Filter (82) + +- Scan parquet (81) (1) Scan parquet @@ -89,392 +94,412 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(10) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(9) Scan parquet +(11) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(12) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(16) Scan parquet +(18) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(17) ProjectExecTransformer +(19) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(20) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(18) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(19) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(20) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(21) InputAdapter +(24) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(22) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(23) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(24) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(25) ProjectExecTransformer +(28) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(26) Scan parquet +(29) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(27) ProjectExecTransformer +(30) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(31) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(28) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastQueryStage +(34) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [1]: [o_orderkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [1]: [o_orderkey#X] -(33) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(36) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(37) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(44) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [1]: [n_nationkey#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [1]: [n_nationkey#X] -(42) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(43) ProjectExecTransformer +(48) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(44) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(45) ProjectExecTransformer +(50) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(46) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(47) ColumnarExchange +(52) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(49) InputAdapter +(54) InputAdapter Input [2]: [s_name#X, count#X] -(50) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(51) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(53) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(54) Scan parquet +(59) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) BroadcastExchange +(61) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(62) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(63) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(59) Project +(64) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(60) Scan parquet +(65) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) BroadcastExchange +(66) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(63) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(64) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(65) Project +(70) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) BroadcastExchange +(71) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(69) Project +(74) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(70) Scan parquet +(75) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(76) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(72) Project +(77) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(73) BroadcastExchange +(78) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(74) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(75) Project +(80) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(76) Scan parquet +(81) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(77) Filter +(82) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(78) Project +(83) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(79) BroadcastExchange +(84) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(81) Project +(86) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(82) HashAggregate +(87) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(83) Exchange +(88) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(89) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(85) TakeOrderedAndProject +(90) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(86) AdaptiveSparkPlan +(91) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 503fe27ab555..698879c473b6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,39 +1,40 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (38) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ SortExecTransformer (23) - +- ^ InputIteratorTransformer (22) - +- ^ InputAdapter (21) - +- ^ ShuffleQueryStage (20), Statistics(X) - +- ColumnarExchange (19) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (7) - +- ^ InputAdapter (6) - +- ^ BroadcastQueryStage (5), Statistics(X) - +- ColumnarBroadcastExchange (4) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (36) - +- Exchange (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin LeftAnti BuildRight (30) - :- Filter (27) - : +- Scan parquet (26) - +- BroadcastExchange (29) - +- Scan parquet (28) + Sort (37) + +- Exchange (36) + +- HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin LeftAnti BuildRight (31) + :- Filter (28) + : +- Scan parquet (27) + +- BroadcastExchange (30) + +- Scan parquet (29) (1) Scan parquet @@ -43,272 +44,281 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) Scan parquet +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [o_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [o_custkey#X] -(8) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(10) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(14) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(19) ColumnarExchange +(20) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(21) InputAdapter +(22) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(22) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) SortExecTransformer +(24) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(24) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(25) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(26) Scan parquet +(27) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(27) Filter +(28) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(28) Scan parquet +(29) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(29) BroadcastExchange +(30) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(31) Project +(32) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(32) HashAggregate +(33) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(35) Exchange +(36) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(37) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(37) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (55) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (48) - +- ^ RegularHashAggregateExecTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ShuffleQueryStage (43), Statistics(X) - +- ColumnarExchange (42) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ Scan parquet (38) + VeloxColumnarToRowExec (50) + +- ^ RegularHashAggregateExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) -(38) Scan parquet +(39) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(39) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(41) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(40) FlushableHashAggregateExecTransformer +(42) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(41) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(42) ColumnarExchange +(44) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(44) InputAdapter +(46) InputAdapter Input [2]: [sum#X, count#X] -(45) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [sum#X, count#X] -(46) RegularHashAggregateExecTransformer +(48) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(48) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(49) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(50) Filter +(52) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(51) Project +(53) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(52) HashAggregate +(54) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(53) Exchange +(55) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index cd21047a80a5..98a7cd0e0145 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,52 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (31) - +- TakeOrderedAndProjectExecTransformer (30) - +- ^ ProjectExecTransformer (28) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ BroadcastHashJoinExecTransformer Inner (18) - :- ^ ProjectExecTransformer (10) - : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ BroadcastQueryStage (15), Statistics(X) - +- ColumnarBroadcastExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ Scan parquet (11) + VeloxColumnarToRowExec (34) + +- TakeOrderedAndProjectExecTransformer (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Project (45) - +- BroadcastHashJoin Inner BuildRight (44) - :- Project (39) - : +- BroadcastHashJoin Inner BuildLeft (38) - : :- BroadcastExchange (35) - : : +- Project (34) - : : +- Filter (33) - : : +- Scan parquet (32) - : +- Filter (37) - : +- Scan parquet (36) - +- BroadcastExchange (43) - +- Project (42) - +- Filter (41) - +- Scan parquet (40) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -56,226 +59,238 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(12) ProjectExecTransformer +(14) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(15) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(15) BroadcastQueryStage +(18) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(19) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(18) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(24) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(26) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(28) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(30) TakeOrderedAndProjectExecTransformer +(33) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(31) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(32) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(33) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(34) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(35) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(36) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(38) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(39) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(41) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(42) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(45) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(46) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(47) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(49) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 95367083053d..b5fefc6bef3c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- ^ SortExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) - :- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- ^ InputAdapter (10) + +- ^ BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin LeftSemi BuildRight (35) - :- Project (30) - : +- Filter (29) - : +- Scan parquet (28) - +- BroadcastExchange (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin LeftSemi BuildRight (37) + :- Project (32) + : +- Filter (31) + : +- Scan parquet (30) + +- BroadcastExchange (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -48,184 +50,192 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) Scan parquet +(4) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(4) ProjectExecTransformer +(5) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(6) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(5) WholeStageCodegenTransformer (X) +(7) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(8) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(9) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(8) InputAdapter +(10) InputAdapter Input [1]: [l_orderkey#X] -(9) InputIteratorTransformer +(11) InputIteratorTransformer Input [1]: [l_orderkey#X] -(10) BroadcastHashJoinExecTransformer +(12) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(12) FlushableHashAggregateExecTransformer +(14) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(13) ProjectExecTransformer +(15) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(14) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(15) ColumnarExchange +(17) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(25) SortExecTransformer +(27) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(28) Scan parquet +(30) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(29) Filter +(31) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(30) Project +(32) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(31) Scan parquet +(33) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(32) Filter +(34) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(33) Project +(35) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(34) BroadcastExchange +(36) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) Project +(38) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(37) HashAggregate +(39) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(38) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index 71c8049fa4dd..be0403f587f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,92 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ SortExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ColumnarExchange (47) - +- ^ ProjectExecTransformer (45) - +- ^ FlushableHashAggregateExecTransformer (44) - +- ^ ProjectExecTransformer (43) - +- ^ BroadcastHashJoinExecTransformer Inner (42) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (8) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ BroadcastQueryStage (39), Statistics(X) - +- ColumnarBroadcastExchange (38) - +- ^ ProjectExecTransformer (36) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (93) - +- Exchange (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (62) - : : : : : +- Filter (61) - : : : : : +- Scan parquet (60) - : : : : +- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Project (71) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -96,424 +102,448 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(8) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(10) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(9) BroadcastHashJoinExecTransformer +(11) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(11) Scan parquet +(13) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(15) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(24) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(27) Scan parquet +(31) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(33) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(36) ProjectExecTransformer +(41) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(42) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(37) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(38) ColumnarBroadcastExchange +(44) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) BroadcastQueryStage +(45) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(40) InputAdapter +(46) InputAdapter Input [1]: [r_regionkey#X] -(41) InputIteratorTransformer +(47) InputIteratorTransformer Input [1]: [r_regionkey#X] -(42) BroadcastHashJoinExecTransformer +(48) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(43) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(44) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(45) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(46) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(47) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(55) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(52) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(53) ColumnarExchange +(59) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(55) InputAdapter +(61) InputAdapter Input [2]: [n_name#X, revenue#X] -(56) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(57) SortExecTransformer +(63) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(60) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(62) BroadcastExchange +(68) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(65) Project +(71) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(66) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(67) Project +(73) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(70) BroadcastExchange +(76) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(72) Project +(78) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(79) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(80) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(81) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(77) Project +(83) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(84) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(80) BroadcastExchange +(86) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(88) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(83) Scan parquet +(89) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(84) Filter +(90) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(85) Project +(91) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(86) BroadcastExchange +(92) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(88) Project +(94) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) HashAggregate +(95) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) Exchange +(96) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(97) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(92) Exchange +(98) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(99) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 733adf0d0b4a..9333e2a8ad59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 93a86cb78a95..2ac2968387fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,87 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ SortExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ RegularHashAggregateExecTransformer (47) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FlushableHashAggregateExecTransformer (40) - +- ^ ProjectExecTransformer (39) - +- ^ BroadcastHashJoinExecTransformer Inner (38) - :- ^ ProjectExecTransformer (33) - : +- ^ BroadcastHashJoinExecTransformer Inner (32) - : :- ^ ProjectExecTransformer (25) - : : +- ^ BroadcastHashJoinExecTransformer Inner (24) - : : :- ^ ProjectExecTransformer (17) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) - : : : :- ^ ProjectExecTransformer (9) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) - : : : : :- ^ InputIteratorTransformer (6) - : : : : : +- ^ InputAdapter (5) - : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (3) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (7) - : : : +- ^ InputIteratorTransformer (15) - : : : +- ^ InputAdapter (14) - : : : +- ^ BroadcastQueryStage (13), Statistics(X) - : : : +- ColumnarBroadcastExchange (12) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ BroadcastQueryStage (21), Statistics(X) - : : +- ColumnarBroadcastExchange (20) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ BroadcastQueryStage (29), Statistics(X) - : +- ColumnarBroadcastExchange (28) - : +- ^ Scan parquet (26) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ BroadcastQueryStage (35), Statistics(X) - +- ReusedExchange (34) + VeloxColumnarToRowExec (60) + +- ^ SortExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- ^ InputAdapter (17) + : : : +- ^ BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- ^ InputAdapter (26) + : : +- ^ BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (87) - +- Exchange (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (77) - : +- BroadcastHashJoin Inner BuildRight (76) - : :- Project (72) - : : +- BroadcastHashJoin Inner BuildRight (71) - : : :- Project (67) - : : : +- BroadcastHashJoin Inner BuildRight (66) - : : : :- Project (62) - : : : : +- BroadcastHashJoin Inner BuildLeft (61) - : : : : :- BroadcastExchange (58) - : : : : : +- Filter (57) - : : : : : +- Scan parquet (56) - : : : : +- Filter (60) - : : : : +- Scan parquet (59) - : : : +- BroadcastExchange (65) - : : : +- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (75) - : +- Filter (74) - : +- Scan parquet (73) - +- BroadcastExchange (80) - +- Filter (79) - +- Scan parquet (78) + Sort (92) + +- Exchange (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (63) + : : : : : +- Filter (62) + : : : : : +- Scan parquet (61) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -91,396 +96,416 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) WholeStageCodegenTransformer (X) +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(3) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(4) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(5) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(6) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(7) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(8) BroadcastHashJoinExecTransformer +(9) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(10) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(9) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) Scan parquet +(12) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(11) WholeStageCodegenTransformer (X) +(13) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(14) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarBroadcastExchange +(15) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(13) BroadcastQueryStage +(16) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) BroadcastHashJoinExecTransformer +(19) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(17) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(18) Scan parquet +(21) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(19) WholeStageCodegenTransformer (X) +(22) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(23) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(20) ColumnarBroadcastExchange +(24) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(21) BroadcastQueryStage +(25) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(22) InputAdapter +(26) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(23) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(24) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(25) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(26) Scan parquet +(30) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(27) WholeStageCodegenTransformer (X) +(31) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(32) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(28) ColumnarBroadcastExchange +(33) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastQueryStage +(34) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(30) InputAdapter +(35) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(31) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(32) BroadcastHashJoinExecTransformer +(37) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(38) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(34) ReusedExchange [Reuses operator id: 28] +(39) ReusedExchange [Reuses operator id: 33] Output [2]: [n_nationkey#X, n_name#X] -(35) BroadcastQueryStage +(40) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(36) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(37) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(38) BroadcastHashJoinExecTransformer +(43) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(39) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(40) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(43) ColumnarExchange +(48) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(49) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(45) InputAdapter +(50) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) InputIteratorTransformer +(51) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(48) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(49) ColumnarExchange +(54) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(55) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(51) InputAdapter +(56) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(52) InputIteratorTransformer +(57) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(53) SortExecTransformer +(58) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(54) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(55) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(56) Scan parquet +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(57) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(58) BroadcastExchange +(63) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(59) Scan parquet +(64) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(61) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(62) Project +(67) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(63) Scan parquet +(68) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(64) Filter +(69) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(65) BroadcastExchange +(70) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(66) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(67) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(68) Scan parquet +(73) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(70) BroadcastExchange +(75) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(72) Project +(77) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(73) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(74) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(75) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(77) Project +(82) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(78) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(79) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(82) Project +(87) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(88) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(86) Exchange +(91) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Sort +(92) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index caddcc769034..65906bb96691 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,117 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ SortExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ShuffleQueryStage (64), Statistics(X) - +- ColumnarExchange (63) - +- ^ ProjectExecTransformer (61) - +- ^ FlushableHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ BroadcastHashJoinExecTransformer Inner (58) - :- ^ ProjectExecTransformer (50) - : +- ^ BroadcastHashJoinExecTransformer Inner (49) - : :- ^ ProjectExecTransformer (42) - : : +- ^ BroadcastHashJoinExecTransformer Inner (41) - : : :- ^ ProjectExecTransformer (34) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : : : :- ^ ProjectExecTransformer (26) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : : : :- ^ ProjectExecTransformer (18) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : : : :- ^ ProjectExecTransformer (10) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (16) - : : : : : +- ^ InputAdapter (15) - : : : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (13) - : : : : : +- ^ Scan parquet (11) - : : : : +- ^ InputIteratorTransformer (24) - : : : : +- ^ InputAdapter (23) - : : : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : : : +- ColumnarBroadcastExchange (21) - : : : : +- ^ Scan parquet (19) - : : : +- ^ InputIteratorTransformer (32) - : : : +- ^ InputAdapter (31) - : : : +- ^ BroadcastQueryStage (30), Statistics(X) - : : : +- ColumnarBroadcastExchange (29) - : : : +- ^ Scan parquet (27) - : : +- ^ InputIteratorTransformer (40) - : : +- ^ InputAdapter (39) - : : +- ^ BroadcastQueryStage (38), Statistics(X) - : : +- ColumnarBroadcastExchange (37) - : : +- ^ Scan parquet (35) - : +- ^ InputIteratorTransformer (48) - : +- ^ InputAdapter (47) - : +- ^ BroadcastQueryStage (46), Statistics(X) - : +- ColumnarBroadcastExchange (45) - : +- ^ Scan parquet (43) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ BroadcastQueryStage (55), Statistics(X) - +- ColumnarBroadcastExchange (54) - +- ^ ProjectExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (84) + +- ^ SortExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- ^ InputAdapter (18) + : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- ^ InputAdapter (27) + : : : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- ^ InputAdapter (36) + : : : +- ^ BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- ^ InputAdapter (54) + : +- ^ BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- BroadcastHashJoin Inner BuildRight (114) - :- Project (109) - : +- BroadcastHashJoin Inner BuildRight (108) - : :- Project (104) - : : +- BroadcastHashJoin Inner BuildRight (103) - : : :- Project (99) - : : : +- BroadcastHashJoin Inner BuildRight (98) - : : : :- Project (94) - : : : : +- BroadcastHashJoin Inner BuildRight (93) - : : : : :- Project (89) - : : : : : +- BroadcastHashJoin Inner BuildRight (88) - : : : : : :- Project (84) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) - : : : : : : :- BroadcastExchange (80) - : : : : : : : +- Project (79) - : : : : : : : +- Filter (78) - : : : : : : : +- Scan parquet (77) - : : : : : : +- Filter (82) - : : : : : : +- Scan parquet (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- Filter (86) - : : : : : +- Scan parquet (85) - : : : : +- BroadcastExchange (92) - : : : : +- Filter (91) - : : : : +- Scan parquet (90) - : : : +- BroadcastExchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- BroadcastExchange (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- BroadcastExchange (107) - : +- Filter (106) - : +- Scan parquet (105) - +- BroadcastExchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (128) + +- Exchange (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- BroadcastHashJoin Inner BuildRight (122) + :- Project (117) + : +- BroadcastHashJoin Inner BuildRight (116) + : :- Project (112) + : : +- BroadcastHashJoin Inner BuildRight (111) + : : :- Project (107) + : : : +- BroadcastHashJoin Inner BuildRight (106) + : : : :- Project (102) + : : : : +- BroadcastHashJoin Inner BuildRight (101) + : : : : :- Project (97) + : : : : : +- BroadcastHashJoin Inner BuildRight (96) + : : : : : :- Project (92) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) + : : : : : : :- BroadcastExchange (88) + : : : : : : : +- Project (87) + : : : : : : : +- Filter (86) + : : : : : : : +- Scan parquet (85) + : : : : : : +- Filter (90) + : : : : : : +- Scan parquet (89) + : : : : : +- BroadcastExchange (95) + : : : : : +- Filter (94) + : : : : : +- Scan parquet (93) + : : : : +- BroadcastExchange (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- BroadcastExchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- BroadcastExchange (110) + : : +- Filter (109) + : : +- Scan parquet (108) + : +- BroadcastExchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- BroadcastExchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,548 +129,580 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(43) Scan parquet +(49) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(44) WholeStageCodegenTransformer (X) +(50) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(51) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(45) ColumnarBroadcastExchange +(52) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastQueryStage +(53) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(47) InputAdapter +(54) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(48) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastHashJoinExecTransformer +(56) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(50) ProjectExecTransformer +(57) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(51) Scan parquet +(58) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(52) ProjectExecTransformer +(59) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(60) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(53) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(54) ColumnarBroadcastExchange +(62) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(55) BroadcastQueryStage +(63) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(56) InputAdapter +(64) InputAdapter Input [1]: [r_regionkey#X] -(57) InputIteratorTransformer +(65) InputIteratorTransformer Input [1]: [r_regionkey#X] -(58) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(67) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(60) FlushableHashAggregateExecTransformer +(68) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(61) ProjectExecTransformer +(69) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(62) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(63) ColumnarExchange +(71) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(64) ShuffleQueryStage +(72) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(65) InputAdapter +(73) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(66) InputIteratorTransformer +(74) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(67) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(68) ProjectExecTransformer +(76) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(69) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(70) ColumnarExchange +(78) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(72) InputAdapter +(80) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(73) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(74) SortExecTransformer +(82) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(75) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(76) VeloxColumnarToRowExec +(84) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(77) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(78) Filter +(86) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(79) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(80) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(90) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(84) Project +(92) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(85) Scan parquet +(93) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(94) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(95) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(97) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(91) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(92) BroadcastExchange +(100) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(93) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(94) Project +(102) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Scan parquet +(103) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(104) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) BroadcastExchange +(105) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(98) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(99) Project +(107) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(100) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(101) Filter +(109) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(102) BroadcastExchange +(110) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(103) BroadcastHashJoin +(111) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(104) Project +(112) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(105) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(106) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(107) BroadcastExchange +(115) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(108) BroadcastHashJoin +(116) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(109) Project +(117) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(110) Scan parquet +(118) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(111) Filter +(119) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(112) Project +(120) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(113) BroadcastExchange +(121) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(114) BroadcastHashJoin +(122) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(115) Project +(123) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(116) HashAggregate +(124) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(117) Exchange +(125) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(126) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(119) Exchange +(127) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(128) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 7f9173ddd5af..e7abd01744e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,90 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ SortExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer Inner (33) - : :- ^ ProjectExecTransformer (26) - : : +- ^ BroadcastHashJoinExecTransformer Inner (25) - : : :- ^ ProjectExecTransformer (18) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ BroadcastQueryStage (22), Statistics(X) - : : +- ColumnarBroadcastExchange (21) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ColumnarBroadcastExchange (29) - : +- ^ Scan parquet (27) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ Scan parquet (35) + VeloxColumnarToRowExec (64) + +- ^ SortExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ^ InputAdapter (60) + +- ^ ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- ^ InputAdapter (18) + : : : +- ^ BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- ^ InputAdapter (36) + : +- ^ BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (91) - +- Exchange (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildRight (75) - : : :- Project (71) - : : : +- BroadcastHashJoin Inner BuildRight (70) - : : : :- Project (66) - : : : : +- BroadcastHashJoin Inner BuildLeft (65) - : : : : :- BroadcastExchange (62) - : : : : : +- Project (61) - : : : : : +- Filter (60) - : : : : : +- Scan parquet (59) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- BroadcastExchange (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (84) - +- Filter (83) - +- Scan parquet (82) + Sort (97) + +- Exchange (96) + +- HashAggregate (95) + +- Exchange (94) + +- HashAggregate (93) + +- Project (92) + +- BroadcastHashJoin Inner BuildRight (91) + :- Project (87) + : +- BroadcastHashJoin Inner BuildRight (86) + : :- Project (82) + : : +- BroadcastHashJoin Inner BuildRight (81) + : : :- Project (77) + : : : +- BroadcastHashJoin Inner BuildRight (76) + : : : :- Project (72) + : : : : +- BroadcastHashJoin Inner BuildLeft (71) + : : : : :- BroadcastExchange (68) + : : : : : +- Project (67) + : : : : : +- Filter (66) + : : : : : +- Scan parquet (65) + : : : : +- Filter (70) + : : : : +- Scan parquet (69) + : : : +- BroadcastExchange (75) + : : : +- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- BroadcastExchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (90) + +- Filter (89) + +- Scan parquet (88) (1) Scan parquet @@ -94,416 +100,440 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(5) BroadcastQueryStage +(6) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) BroadcastHashJoinExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(10) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) Scan parquet +(13) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(14) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(15) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(16) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(14) BroadcastQueryStage +(17) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(18) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) BroadcastHashJoinExecTransformer +(20) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(21) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(19) Scan parquet +(22) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(20) WholeStageCodegenTransformer (X) +(23) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(24) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(21) ColumnarBroadcastExchange +(25) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(22) BroadcastQueryStage +(26) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(23) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(24) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(25) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(26) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(27) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(28) WholeStageCodegenTransformer (X) +(32) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(29) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(36) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(32) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(33) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(35) Scan parquet +(40) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(36) WholeStageCodegenTransformer (X) +(41) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(42) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(37) ColumnarBroadcastExchange +(43) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(44) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(39) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(40) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(41) BroadcastHashJoinExecTransformer +(47) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(43) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(52) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(56) SortExecTransformer +(62) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(59) Scan parquet +(65) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(60) Filter +(66) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(61) Project +(67) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(62) BroadcastExchange +(68) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) Scan parquet +(69) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(70) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(66) Project +(72) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(67) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(68) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(69) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(70) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(71) Project +(77) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(72) Scan parquet +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(73) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(74) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(75) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(76) Project +(82) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(77) Scan parquet +(83) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(84) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(79) BroadcastExchange +(85) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(81) Project +(87) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(82) Scan parquet +(88) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(84) BroadcastExchange +(90) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(85) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(92) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(87) HashAggregate +(93) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(88) Exchange +(94) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(95) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(90) Exchange +(96) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Sort +(97) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(92) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 699c17ea4562..8d67aad16c3d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 35c0d13f1e1e..9af231ec1bb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,81 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- TakeOrderedAndProjectExecTransformer (55) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ ShuffledHashJoinExecTransformer Inner (43) - :- ^ InputIteratorTransformer (35) - : +- ^ InputAdapter (34) - : +- ^ ShuffleQueryStage (33) - : +- ColumnarExchange (32) - : +- ^ ProjectExecTransformer (30) - : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : :- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ ShuffleQueryStage (26) - : +- ColumnarExchange (25) - : +- ^ ProjectExecTransformer (23) - : +- ^ Scan parquet (22) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ ShuffleQueryStage (40) - +- ColumnarExchange (39) - +- ^ ProjectExecTransformer (37) - +- ^ Scan parquet (36) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- ShuffledHashJoin Inner BuildRight (77) - :- Exchange (73) - : +- Project (72) - : +- ShuffledHashJoin Inner BuildRight (71) - : :- Exchange (66) - : : +- Project (65) - : : +- ShuffledHashJoin Inner BuildRight (64) - : : :- Exchange (59) - : : : +- Filter (58) - : : : +- Scan parquet (57) - : : +- Exchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Exchange (70) - : +- Project (69) - : +- Filter (68) - : +- Scan parquet (67) - +- Exchange (76) - +- Filter (75) - +- Scan parquet (74) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -85,354 +89,370 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(53) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(54) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(55) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(56) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(57) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(58) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(59) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(61) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(62) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(63) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(66) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(68) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(69) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(70) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(73) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(75) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(76) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(79) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(80) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(82) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(83) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 23cd40b681e6..f028b8daa9b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,68 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ^ InputAdapter (43) - +- ^ ShuffleQueryStage (42) - +- ColumnarExchange (41) - +- ^ FilterExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35) - +- ColumnarExchange (34) - +- ^ ProjectExecTransformer (32) - +- ^ FlushableHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ ShuffleQueryStage (38) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- Filter (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- ShuffledHashJoin Inner BuildRight (61) - :- Exchange (56) - : +- Project (55) - : +- ShuffledHashJoin Inner BuildRight (54) - : :- Exchange (50) - : : +- Filter (49) - : : +- Scan parquet (48) - : +- Exchange (53) - : +- Filter (52) - : +- Scan parquet (51) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + Sort (71) + +- Exchange (70) + +- Filter (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- ShuffledHashJoin Inner BuildRight (64) + :- Exchange (59) + : +- Project (58) + : +- ShuffledHashJoin Inner BuildRight (57) + : :- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Exchange (56) + : +- Filter (55) + : +- Scan parquet (54) + +- Exchange (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) (1) Scan parquet @@ -72,292 +75,304 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [1]: [n_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(31) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(34) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(38) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(39) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(40) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(41) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(43) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(44) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(45) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(46) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(47) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(48) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(49) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(50) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(53) Exchange +(56) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(56) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Scan parquet +(60) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(58) Filter +(61) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(59) Project +(62) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) Exchange +(63) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(65) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(63) HashAggregate +(66) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(64) Exchange +(67) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(68) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(66) Filter +(69) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(67) Exchange +(70) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(71) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(69) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index c5f7a17f4286..914b1813df44 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,48 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- HashAggregate (44) - +- Exchange (43) - +- HashAggregate (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin Inner BuildLeft (42) + :- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -52,200 +54,208 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(35) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(38) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(37) Filter +(39) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(38) Project +(40) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(39) Exchange +(41) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(41) Project +(43) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(42) HashAggregate +(44) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(43) Exchange +(45) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) HashAggregate +(46) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(45) Exchange +(47) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(48) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index 507966beca53..408b8a0f7f97 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftOuter BuildRight (42) - :- Exchange (37) - : +- Scan parquet (36) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftOuter BuildRight (43) + :- Exchange (38) + : +- Scan parquet (37) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -84,198 +85,202 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index fd6ee41dbfa9..c1e2a1e52130 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (32) - +- HashAggregate (31) - +- Project (30) - +- ShuffledHashJoin Inner BuildRight (29) - :- Exchange (25) - : +- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- Exchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- HashAggregate (33) + +- Project (32) + +- ShuffledHashJoin Inner BuildRight (31) + :- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -40,148 +42,156 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(23) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(24) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Scan parquet +(28) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) Exchange +(30) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) ShuffledHashJoin +(31) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(32) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(33) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 66d138fef6d1..1da11e4fffea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,45 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (43) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner (19) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner (21) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (42) - +- Exchange (41) - +- Project (40) - +- ShuffledHashJoin Inner BuildLeft (39) - :- Exchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Filter (38) - +- HashAggregate (37) - +- Exchange (36) - +- HashAggregate (35) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -49,186 +51,194 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(19) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(29) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(31) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(34) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(36) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(38) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(39) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(40) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(41) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(43) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 8acd437dd729..4a8b56e41f36 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35) - +- ColumnarExchange (34) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FlushableHashAggregateExecTransformer (25) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ^ InputAdapter (38) + +- ^ ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- ShuffledHashJoin Inner BuildRight (52) - :- Exchange (48) - : +- BroadcastHashJoin LeftAnti BuildRight (47) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Exchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (63) + +- Exchange (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -66,274 +68,282 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(28) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(30) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(33) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(34) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(36) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(39) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(40) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(43) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(44) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(45) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(46) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(48) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Scan parquet +(51) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(52) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) Exchange +(53) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(55) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(56) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(58) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(63) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index a759dc4f95c1..523c081b1d19 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ ShuffledHashJoinExecTransformer Inner (28) - :- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ FilterExecTransformer (27) - +- ^ ProjectExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ Scan parquet (17) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (53) - +- HashAggregate (52) - +- Project (51) - +- ShuffledHashJoin Inner BuildRight (50) - :- Project (43) - : +- ShuffledHashJoin Inner BuildRight (42) - : :- Exchange (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Filter (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Filter (45) - +- Scan parquet (44) + HashAggregate (56) + +- HashAggregate (55) + +- Project (54) + +- ShuffledHashJoin Inner BuildRight (53) + :- Project (46) + : +- ShuffledHashJoin Inner BuildRight (45) + : :- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Exchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -60,250 +63,262 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_container#X] + +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) NoopFilter +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: [l_partkey#X, l_quantity#X] + +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(20) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(21) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(30) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(37) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(40) Project +(43) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(41) Exchange +(44) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(43) Project +(46) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(45) Filter +(48) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(46) HashAggregate +(49) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(47) Exchange +(50) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(49) Filter +(52) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(50) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(51) Project +(54) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) HashAggregate +(55) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(53) HashAggregate +(56) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(54) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 7a6e11258914..6045020cb49a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,93 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- TakeOrderedAndProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ ShuffledHashJoinExecTransformer Inner (55) - :- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37) - : +- ColumnarExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ ShuffledHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ ShuffleQueryStage (30) - : +- ColumnarExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) - : :- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ ProjectExecTransformer (25) - : +- ^ FilterExecTransformer (24) - : +- ^ RegularHashAggregateExecTransformer (23) - : +- ^ InputIteratorTransformer (22) - : +- ^ InputAdapter (21) - : +- ^ ShuffleQueryStage (20) - : +- ColumnarExchange (19) - : +- ^ ProjectExecTransformer (17) - : +- ^ FlushableHashAggregateExecTransformer (16) - : +- ^ Scan parquet (15) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) - :- ^ InputIteratorTransformer (46) - : +- ^ InputAdapter (45) - : +- ^ ShuffleQueryStage (44) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ Scan parquet (40) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48) - +- ReusedExchange (47) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- HashAggregate (91) - +- Project (90) - +- ShuffledHashJoin Inner BuildRight (89) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildLeft (76) - : :- Exchange (64) - : : +- Filter (63) - : : +- Scan parquet (62) - : +- Exchange (75) - : +- ShuffledHashJoin LeftSemi BuildRight (74) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Project (73) - : +- Filter (72) - : +- HashAggregate (71) - : +- Exchange (70) - : +- HashAggregate (69) - : +- Scan parquet (68) - +- ShuffledHashJoin LeftSemi BuildRight (88) - :- Exchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- Project (87) - +- Filter (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Scan parquet (82) + TakeOrderedAndProject (96) + +- HashAggregate (95) + +- HashAggregate (94) + +- Project (93) + +- ShuffledHashJoin Inner BuildRight (92) + :- Exchange (81) + : +- Project (80) + : +- ShuffledHashJoin Inner BuildLeft (79) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (78) + : +- ShuffledHashJoin LeftSemi BuildRight (77) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Project (76) + : +- Filter (75) + : +- HashAggregate (74) + : +- Exchange (73) + : +- HashAggregate (72) + : +- Scan parquet (71) + +- ShuffledHashJoin LeftSemi BuildRight (91) + :- Exchange (84) + : +- Filter (83) + : +- Scan parquet (82) + +- Project (90) + +- Filter (89) + +- HashAggregate (88) + +- Exchange (87) + +- HashAggregate (86) + +- Scan parquet (85) (1) Scan parquet @@ -97,420 +100,432 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(19) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(21) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(23) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(24) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(25) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(26) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(29) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(34) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(36) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(38) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(39) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(40) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(47) ReusedExchange [Reuses operator id: 19] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(48) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(52) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(54) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(57) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(58) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(59) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(60) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(61) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(62) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(63) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(64) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(68) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(69) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(67) Exchange +(70) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(75) Exchange +(78) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(79) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(77) Project +(80) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(78) Exchange +(81) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(81) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(83) HashAggregate +(86) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(84) Exchange +(87) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(88) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(86) Filter +(89) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(87) Project +(90) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(88) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(89) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(90) Project +(93) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(91) HashAggregate +(94) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(92) HashAggregate +(95) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(93) TakeOrderedAndProject +(96) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(94) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index 260269c9eaef..7084e61f61d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (31) - +- HashAggregate (30) - +- Project (29) - +- ShuffledHashJoin Inner BuildRight (28) - :- Exchange (24) - : +- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- Exchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- HashAggregate (32) + +- Project (31) + +- ShuffledHashJoin Inner BuildRight (30) + :- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Exchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(21) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(22) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(23) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(24) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) Scan parquet +(27) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) Exchange +(29) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) ShuffledHashJoin +(30) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(31) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(32) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 2071bbf73f12..3f6d17512477 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,116 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (121) +AdaptiveSparkPlan (126) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ SortExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ ShuffledHashJoinExecTransformer Inner (72) - :- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ ShuffleQueryStage (62) - : +- ColumnarExchange (61) - : +- ^ ProjectExecTransformer (59) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ ShuffleQueryStage (26) - : : +- ColumnarExchange (25) - : : +- ^ ProjectExecTransformer (23) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : :- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ ShuffleQueryStage (12) - : : : +- ColumnarExchange (11) - : : : +- ^ ProjectExecTransformer (9) - : : : +- ^ Scan parquet (8) - : : +- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ Scan parquet (15) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) - : :- ^ InputIteratorTransformer (35) - : : +- ^ InputAdapter (34) - : : +- ^ ShuffleQueryStage (33) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ Scan parquet (29) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37) - : +- ReusedExchange (36) - +- ^ InputIteratorTransformer (71) - +- ^ InputAdapter (70) - +- ^ ShuffleQueryStage (69) - +- ColumnarExchange (68) - +- ^ ProjectExecTransformer (66) - +- ^ Scan parquet (65) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ^ InputAdapter (82) + +- ^ ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (31) + : : +- ^ InputAdapter (30) + : : +- ^ ShuffleQueryStage (29) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ ShuffleQueryStage (14) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ ShuffleQueryStage (22) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : :- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ ShuffleQueryStage (41) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (120) - +- Exchange (119) - +- Project (118) - +- ShuffledHashJoin Inner BuildRight (117) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin LeftSemi BuildRight (110) - : :- Exchange (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin Inner BuildLeft (107) - : :- Exchange (93) - : : +- ShuffledHashJoin LeftSemi BuildRight (92) - : : :- Exchange (87) - : : : +- Filter (86) - : : : +- Scan parquet (85) - : : +- Exchange (91) - : : +- Project (90) - : : +- Filter (89) - : : +- Scan parquet (88) - : +- Exchange (106) - : +- Filter (105) - : +- HashAggregate (104) - : +- HashAggregate (103) - : +- ShuffledHashJoin LeftSemi BuildRight (102) - : :- Exchange (97) - : : +- Project (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (101) - : +- Project (100) - : +- Filter (99) - : +- Scan parquet (98) - +- Exchange (116) - +- Project (115) - +- Filter (114) - +- Scan parquet (113) + Sort (125) + +- Exchange (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin LeftSemi BuildRight (115) + : :- Exchange (89) + : : +- Filter (88) + : : +- Scan parquet (87) + : +- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin Inner BuildLeft (112) + : :- Exchange (98) + : : +- ShuffledHashJoin LeftSemi BuildRight (97) + : : :- Exchange (92) + : : : +- Filter (91) + : : : +- Scan parquet (90) + : : +- Exchange (96) + : : +- Project (95) + : : +- Filter (94) + : : +- Scan parquet (93) + : +- Exchange (111) + : +- Filter (110) + : +- HashAggregate (109) + : +- HashAggregate (108) + : +- ShuffledHashJoin LeftSemi BuildRight (107) + : :- Exchange (102) + : : +- Project (101) + : : +- Filter (100) + : : +- Scan parquet (99) + : +- Exchange (106) + : +- Project (105) + : +- Filter (104) + : +- Scan parquet (103) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -120,510 +125,530 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(16) ProjectExecTransformer +(18) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(17) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(18) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(23) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(29) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(30) ProjectExecTransformer +(33) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(31) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(32) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(34) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(35) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(36) ReusedExchange [Reuses operator id: 18] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(37) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [p_partkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -(40) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(58) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(59) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(60) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(61) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(64) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(65) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(66) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(67) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(68) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(69) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(74) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(75) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(77) InputAdapter +(82) InputAdapter Input [2]: [s_name#X, s_address#X] -(78) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(79) SortExecTransformer +(84) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(80) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(81) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(82) Scan parquet +(87) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(84) Exchange +(89) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(87) Exchange +(92) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(93) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(90) Project +(95) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(91) Exchange +(96) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(93) Exchange +(98) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(99) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(95) Filter +(100) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(96) Project +(101) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(97) Exchange +(102) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(99) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(100) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(101) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(103) HashAggregate +(108) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(104) HashAggregate +(109) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(105) Filter +(110) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(106) Exchange +(111) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(108) Project +(113) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(109) Exchange +(114) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(111) Project +(116) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(112) Exchange +(117) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(115) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(116) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(118) Project +(123) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(119) Exchange +(124) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(125) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(121) AdaptiveSparkPlan +(126) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index c4f7c527a056..b7067f8b5e3a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,109 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (114) +AdaptiveSparkPlan (119) +- == Final Plan == - VeloxColumnarToRowExec (77) - +- TakeOrderedAndProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ FlushableHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner (65) - :- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (43) - : : +- ^ InputAdapter (42) - : : +- ^ ShuffleQueryStage (41) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer Inner (37) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (36) - : : +- ^ InputAdapter (35) - : : +- ^ ShuffleQueryStage (34) - : : +- ColumnarExchange (33) - : : +- ^ ProjectExecTransformer (31) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : : :- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (21) - : : : +- ^ InputAdapter (20) - : : : +- ^ ShuffleQueryStage (19) - : : : +- ColumnarExchange (18) - : : : +- ^ ProjectExecTransformer (16) - : : : +- ^ Scan parquet (15) - : : +- ^ InputIteratorTransformer (29) - : : +- ^ InputAdapter (28) - : : +- ^ ShuffleQueryStage (27) - : : +- ColumnarExchange (26) - : : +- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ Scan parquet (44) - +- ^ InputIteratorTransformer (64) - +- ^ InputAdapter (63) - +- ^ ShuffleQueryStage (62) - +- ColumnarExchange (61) - +- ^ ProjectExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (82) + +- TakeOrderedAndProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (113) - +- HashAggregate (112) - +- Exchange (111) - +- HashAggregate (110) - +- Project (109) - +- ShuffledHashJoin Inner BuildRight (108) - :- Exchange (103) - : +- Project (102) - : +- ShuffledHashJoin Inner BuildRight (101) - : :- Exchange (96) - : : +- Project (95) - : : +- ShuffledHashJoin Inner BuildLeft (94) - : : :- Exchange (80) - : : : +- Filter (79) - : : : +- Scan parquet (78) - : : +- Exchange (93) - : : +- ShuffledHashJoin LeftAnti BuildRight (92) - : : :- ShuffledHashJoin LeftSemi BuildRight (87) - : : : :- Exchange (84) - : : : : +- Project (83) - : : : : +- Filter (82) - : : : : +- Scan parquet (81) - : : : +- Exchange (86) - : : : +- Scan parquet (85) - : : +- Exchange (91) - : : +- Project (90) - : : +- Filter (89) - : : +- Scan parquet (88) - : +- Exchange (100) - : +- Project (99) - : +- Filter (98) - : +- Scan parquet (97) - +- Exchange (107) - +- Project (106) - +- Filter (105) - +- Scan parquet (104) + TakeOrderedAndProject (118) + +- HashAggregate (117) + +- Exchange (116) + +- HashAggregate (115) + +- Project (114) + +- ShuffledHashJoin Inner BuildRight (113) + :- Exchange (108) + : +- Project (107) + : +- ShuffledHashJoin Inner BuildRight (106) + : :- Exchange (101) + : : +- Project (100) + : : +- ShuffledHashJoin Inner BuildLeft (99) + : : :- Exchange (85) + : : : +- Filter (84) + : : : +- Scan parquet (83) + : : +- Exchange (98) + : : +- ShuffledHashJoin LeftAnti BuildRight (97) + : : :- ShuffledHashJoin LeftSemi BuildRight (92) + : : : :- Exchange (89) + : : : : +- Project (88) + : : : : +- Filter (87) + : : : : +- Scan parquet (86) + : : : +- Exchange (91) + : : : +- Scan parquet (90) + : : +- Exchange (96) + : : +- Project (95) + : : +- Filter (94) + : : +- Scan parquet (93) + : +- Exchange (105) + : +- Project (104) + : +- Filter (103) + : +- Scan parquet (102) + +- Exchange (112) + +- Project (111) + +- Filter (110) + +- Scan parquet (109) (1) Scan parquet @@ -113,486 +118,506 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(22) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(23) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(24) ProjectExecTransformer +(26) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(25) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(26) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(28) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(29) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(30) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(31) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(33) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(35) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(36) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(37) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(38) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(40) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(42) InputAdapter +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(43) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(45) ProjectExecTransformer +(48) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(58) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(59) ProjectExecTransformer +(63) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(61) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(63) InputAdapter +(68) InputAdapter Input [1]: [n_nationkey#X] -(64) InputIteratorTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -(65) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(66) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(67) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(68) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(69) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(70) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(72) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(73) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(74) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(75) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(76) TakeOrderedAndProjectExecTransformer +(81) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(77) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(78) Scan parquet +(83) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(79) Filter +(84) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(80) Exchange +(85) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) Scan parquet +(86) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(82) Filter +(87) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(83) Project +(88) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) Exchange +(91) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(88) Scan parquet +(93) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(89) Filter +(94) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(90) Project +(95) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(91) Exchange +(96) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(95) Project +(100) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(96) Exchange +(101) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Scan parquet +(102) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(98) Filter +(103) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(99) Project +(104) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(100) Exchange +(105) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(107) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(103) Exchange +(108) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(105) Filter +(110) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(106) Project +(111) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(107) Exchange +(112) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) Project +(114) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(110) HashAggregate +(115) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(111) Exchange +(116) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) HashAggregate +(117) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(113) TakeOrderedAndProject +(118) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(114) AdaptiveSparkPlan +(119) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 0f13ad3079c9..dc38dbb7675a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ^ InputAdapter (29) + +- ^ ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- ShuffledHashJoin LeftAnti BuildRight (38) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (37) - +- Scan parquet (36) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- ShuffledHashJoin LeftAnti BuildRight (39) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Exchange (38) + +- Scan parquet (37) (1) Scan parquet @@ -50,190 +51,194 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(17) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(20) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(35) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(40) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(40) HashAggregate +(41) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(43) Exchange +(44) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index 318b9bdf6a46..d6c6f13e9063 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,57 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- TakeOrderedAndProjectExecTransformer (35) - +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Project (52) - +- ShuffledHashJoin Inner BuildRight (51) - :- Exchange (46) - : +- Project (45) - : +- ShuffledHashJoin Inner BuildLeft (44) - : :- Exchange (40) - : : +- Project (39) - : : +- Filter (38) - : : +- Scan parquet (37) - : +- Exchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Exchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -61,244 +64,256 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(32) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(33) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(37) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(38) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(39) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(40) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(43) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(45) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(46) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(49) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(52) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(53) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(54) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(55) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 807c29568ead..4c33fb73f757 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,49 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- ShuffledHashJoin LeftSemi BuildRight (41) - :- Exchange (36) - : +- Project (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftSemi BuildRight (43) + :- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -53,204 +55,212 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(35) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(36) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(39) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(38) Filter +(40) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(39) Project +(41) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(40) Exchange +(42) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) Project +(44) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(43) HashAggregate +(45) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(44) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(46) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index fc394df77026..8bc2587dbaee 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,121 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (134) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- ShuffledHashJoin Inner BuildRight (127) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (126) + +- Project (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -125,540 +131,564 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [1]: [r_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(91) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(93) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(94) Project +(100) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Exchange +(101) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(97) Project +(103) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(98) Exchange +(104) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(101) Exchange +(107) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(103) Project +(109) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(110) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(109) Project +(115) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(116) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(113) Exchange +(119) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(121) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(116) Exchange +(122) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(119) Project +(125) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(120) Exchange +(126) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(127) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(122) Project +(128) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(123) HashAggregate +(129) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(124) Exchange +(130) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(131) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(126) Exchange +(132) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(133) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(128) AdaptiveSparkPlan +(134) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index cc87fe6d2bbf..8d1a71e9751e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index b9b72a9de06a..5ac66a7e2708 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,117 +1,122 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ^ InputAdapter (81) - +- ^ ShuffleQueryStage (80) - +- ColumnarExchange (79) - +- ^ RegularHashAggregateExecTransformer (77) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FlushableHashAggregateExecTransformer (70) - +- ^ ProjectExecTransformer (69) - +- ^ ShuffledHashJoinExecTransformer Inner (68) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65) - +- ReusedExchange (64) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ^ InputAdapter (86) + +- ^ ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner (73) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ^ InputAdapter (71) + +- ^ ShuffleQueryStage (70) + +- ReusedExchange (69) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- HashAggregate (120) - +- Exchange (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin Inner BuildRight (110) - : :- Exchange (106) - : : +- Project (105) - : : +- ShuffledHashJoin Inner BuildRight (104) - : : :- Exchange (100) - : : : +- Project (99) - : : : +- ShuffledHashJoin Inner BuildRight (98) - : : : :- Exchange (94) - : : : : +- Project (93) - : : : : +- ShuffledHashJoin Inner BuildLeft (92) - : : : : :- Exchange (88) - : : : : : +- Filter (87) - : : : : : +- Scan parquet (86) - : : : : +- Exchange (91) - : : : : +- Filter (90) - : : : : +- Scan parquet (89) - : : : +- Exchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- Exchange (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (109) - : +- Filter (108) - : +- Scan parquet (107) - +- Exchange (115) - +- Filter (114) - +- Scan parquet (113) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildRight (109) + : : :- Exchange (105) + : : : +- Project (104) + : : : +- ShuffledHashJoin Inner BuildRight (103) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- ShuffledHashJoin Inner BuildLeft (97) + : : : : :- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,516 +126,536 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(64) ReusedExchange [Reuses operator id: 53] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(65) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(66) InputAdapter +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(67) InputIteratorTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(68) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(69) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(70) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(71) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(72) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(73) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(75) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(78) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(79) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(81) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(82) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(83) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(85) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(86) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(94) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(95) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) Exchange +(96) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(93) Project +(98) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(97) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(99) Project +(104) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(100) Exchange +(105) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(102) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(103) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(105) Project +(110) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(106) Exchange +(111) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(108) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(109) Exchange +(114) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(111) Project +(116) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(112) Exchange +(117) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(115) Exchange +(120) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(117) Project +(122) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(118) HashAggregate +(123) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(119) Exchange +(124) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) HashAggregate +(125) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(121) Exchange +(126) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(127) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 38e775dcd2ef..7e7064f9fbae 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,158 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (169) +AdaptiveSparkPlan (177) +- == Final Plan == - VeloxColumnarToRowExec (117) - +- ^ SortExecTransformer (115) - +- ^ InputIteratorTransformer (114) - +- ^ InputAdapter (113) - +- ^ ShuffleQueryStage (112) - +- ColumnarExchange (111) - +- ^ ProjectExecTransformer (109) - +- ^ RegularHashAggregateExecTransformer (108) - +- ^ InputIteratorTransformer (107) - +- ^ InputAdapter (106) - +- ^ ShuffleQueryStage (105) - +- ColumnarExchange (104) - +- ^ ProjectExecTransformer (102) - +- ^ FlushableHashAggregateExecTransformer (101) - +- ^ ProjectExecTransformer (100) - +- ^ ShuffledHashJoinExecTransformer Inner (99) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (77) - : : +- ^ InputAdapter (76) - : : +- ^ ShuffleQueryStage (75) - : : +- ColumnarExchange (74) - : : +- ^ ProjectExecTransformer (72) - : : +- ^ ShuffledHashJoinExecTransformer Inner (71) - : : :- ^ InputIteratorTransformer (63) - : : : +- ^ InputAdapter (62) - : : : +- ^ ShuffleQueryStage (61) - : : : +- ColumnarExchange (60) - : : : +- ^ ProjectExecTransformer (58) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : : : :- ^ InputIteratorTransformer (49) - : : : : +- ^ InputAdapter (48) - : : : : +- ^ ShuffleQueryStage (47) - : : : : +- ColumnarExchange (46) - : : : : +- ^ ProjectExecTransformer (44) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : : : :- ^ InputIteratorTransformer (35) - : : : : : +- ^ InputAdapter (34) - : : : : : +- ^ ShuffleQueryStage (33) - : : : : : +- ColumnarExchange (32) - : : : : : +- ^ ProjectExecTransformer (30) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : : : :- ^ InputIteratorTransformer (21) - : : : : : : +- ^ InputAdapter (20) - : : : : : : +- ^ ShuffleQueryStage (19) - : : : : : : +- ColumnarExchange (18) - : : : : : : +- ^ ProjectExecTransformer (16) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ ShuffleQueryStage (5) - : : : : : : : +- ColumnarExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (14) - : : : : : : +- ^ InputAdapter (13) - : : : : : : +- ^ ShuffleQueryStage (12) - : : : : : : +- ColumnarExchange (11) - : : : : : : +- ^ ProjectExecTransformer (9) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (28) - : : : : : +- ^ InputAdapter (27) - : : : : : +- ^ ShuffleQueryStage (26) - : : : : : +- ColumnarExchange (25) - : : : : : +- ^ ProjectExecTransformer (23) - : : : : : +- ^ Scan parquet (22) - : : : : +- ^ InputIteratorTransformer (42) - : : : : +- ^ InputAdapter (41) - : : : : +- ^ ShuffleQueryStage (40) - : : : : +- ColumnarExchange (39) - : : : : +- ^ ProjectExecTransformer (37) - : : : : +- ^ Scan parquet (36) - : : : +- ^ InputIteratorTransformer (56) - : : : +- ^ InputAdapter (55) - : : : +- ^ ShuffleQueryStage (54) - : : : +- ColumnarExchange (53) - : : : +- ^ ProjectExecTransformer (51) - : : : +- ^ Scan parquet (50) - : : +- ^ InputIteratorTransformer (70) - : : +- ^ InputAdapter (69) - : : +- ^ ShuffleQueryStage (68) - : : +- ColumnarExchange (67) - : : +- ^ ProjectExecTransformer (65) - : : +- ^ Scan parquet (64) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82) - : +- ColumnarExchange (81) - : +- ^ ProjectExecTransformer (79) - : +- ^ Scan parquet (78) - +- ^ InputIteratorTransformer (98) - +- ^ InputAdapter (97) - +- ^ ShuffleQueryStage (96) - +- ColumnarExchange (95) - +- ^ ProjectExecTransformer (93) - +- ^ Scan parquet (92) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ^ InputAdapter (121) + +- ^ ShuffleQueryStage (120) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ^ InputAdapter (114) + +- ^ ShuffleQueryStage (113) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner (107) + :- ^ InputIteratorTransformer (98) + : +- ^ InputAdapter (97) + : +- ^ ShuffleQueryStage (96) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : :- ^ InputIteratorTransformer (83) + : : +- ^ InputAdapter (82) + : : +- ^ ShuffleQueryStage (81) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ^ InputAdapter (67) + : : : +- ^ ShuffleQueryStage (66) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ^ InputAdapter (52) + : : : : +- ^ ShuffleQueryStage (51) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ^ InputAdapter (37) + : : : : : +- ^ ShuffleQueryStage (36) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ^ InputAdapter (22) + : : : : : : +- ^ ShuffleQueryStage (21) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ ShuffleQueryStage (6) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ^ InputAdapter (15) + : : : : : : +- ^ ShuffleQueryStage (14) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ^ InputAdapter (30) + : : : : : +- ^ ShuffleQueryStage (29) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ^ InputAdapter (45) + : : : : +- ^ ShuffleQueryStage (44) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ^ InputAdapter (60) + : : : +- ^ ShuffleQueryStage (59) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ^ InputAdapter (105) + +- ^ ShuffleQueryStage (104) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (168) - +- Exchange (167) - +- HashAggregate (166) - +- Exchange (165) - +- HashAggregate (164) - +- Project (163) - +- ShuffledHashJoin Inner BuildRight (162) - :- Exchange (157) - : +- Project (156) - : +- ShuffledHashJoin Inner BuildRight (155) - : :- Exchange (151) - : : +- Project (150) - : : +- ShuffledHashJoin Inner BuildRight (149) - : : :- Exchange (145) - : : : +- Project (144) - : : : +- ShuffledHashJoin Inner BuildRight (143) - : : : :- Exchange (139) - : : : : +- Project (138) - : : : : +- ShuffledHashJoin Inner BuildRight (137) - : : : : :- Exchange (133) - : : : : : +- Project (132) - : : : : : +- ShuffledHashJoin Inner BuildRight (131) - : : : : : :- Exchange (127) - : : : : : : +- Project (126) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) - : : : : : : :- Exchange (121) - : : : : : : : +- Project (120) - : : : : : : : +- Filter (119) - : : : : : : : +- Scan parquet (118) - : : : : : : +- Exchange (124) - : : : : : : +- Filter (123) - : : : : : : +- Scan parquet (122) - : : : : : +- Exchange (130) - : : : : : +- Filter (129) - : : : : : +- Scan parquet (128) - : : : : +- Exchange (136) - : : : : +- Filter (135) - : : : : +- Scan parquet (134) - : : : +- Exchange (142) - : : : +- Filter (141) - : : : +- Scan parquet (140) - : : +- Exchange (148) - : : +- Filter (147) - : : +- Scan parquet (146) - : +- Exchange (154) - : +- Filter (153) - : +- Scan parquet (152) - +- Exchange (161) - +- Project (160) - +- Filter (159) - +- Scan parquet (158) + Sort (176) + +- Exchange (175) + +- HashAggregate (174) + +- Exchange (173) + +- HashAggregate (172) + +- Project (171) + +- ShuffledHashJoin Inner BuildRight (170) + :- Exchange (165) + : +- Project (164) + : +- ShuffledHashJoin Inner BuildRight (163) + : :- Exchange (159) + : : +- Project (158) + : : +- ShuffledHashJoin Inner BuildRight (157) + : : :- Exchange (153) + : : : +- Project (152) + : : : +- ShuffledHashJoin Inner BuildRight (151) + : : : :- Exchange (147) + : : : : +- Project (146) + : : : : +- ShuffledHashJoin Inner BuildRight (145) + : : : : :- Exchange (141) + : : : : : +- Project (140) + : : : : : +- ShuffledHashJoin Inner BuildRight (139) + : : : : : :- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) + : : : : : : :- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Exchange (132) + : : : : : : +- Filter (131) + : : : : : : +- Scan parquet (130) + : : : : : +- Exchange (138) + : : : : : +- Filter (137) + : : : : : +- Scan parquet (136) + : : : : +- Exchange (144) + : : : : +- Filter (143) + : : : : +- Scan parquet (142) + : : : +- Exchange (150) + : : : +- Filter (149) + : : : +- Scan parquet (148) + : : +- Exchange (156) + : : +- Filter (155) + : : +- Scan parquet (154) + : +- Exchange (162) + : +- Filter (161) + : +- Scan parquet (160) + +- Exchange (169) + +- Project (168) + +- Filter (167) + +- Scan parquet (166) (1) Scan parquet @@ -162,712 +170,744 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(73) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(74) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(76) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(77) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) ProjectExecTransformer +(85) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(80) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(81) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(92) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(93) ProjectExecTransformer +(100) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(94) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(95) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(97) InputAdapter +(105) InputAdapter Input [1]: [r_regionkey#X] -(98) InputIteratorTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -(99) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(101) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(102) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(103) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(104) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(105) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(106) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(107) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(108) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(109) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(110) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(111) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(112) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(113) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(114) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(115) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(116) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(117) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(118) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(120) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(121) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(130) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(123) Filter +(131) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(124) Exchange +(132) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(133) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(126) Project +(134) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(127) Exchange +(135) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(129) Filter +(137) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(130) Exchange +(138) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(132) Project +(140) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(133) Exchange +(141) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(142) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(135) Filter +(143) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(136) Exchange +(144) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(145) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(138) Project +(146) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(139) Exchange +(147) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Scan parquet +(148) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(141) Filter +(149) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(142) Exchange +(150) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) ShuffledHashJoin +(151) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(144) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(145) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(154) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(147) Filter +(155) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(148) Exchange +(156) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(157) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(150) Project +(158) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(151) Exchange +(159) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(160) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(153) Filter +(161) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(154) Exchange +(162) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(163) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(156) Project +(164) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(157) Exchange +(165) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(166) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(159) Filter +(167) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(160) Project +(168) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(161) Exchange +(169) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(162) ShuffledHashJoin +(170) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(163) Project +(171) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(164) HashAggregate +(172) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(165) Exchange +(173) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) HashAggregate +(174) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(167) Exchange +(175) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(176) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(169) AdaptiveSparkPlan +(177) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 8ba93a20d0cd..5ba4a9f7ce0e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,120 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (126) - +- Exchange (125) - +- HashAggregate (124) - +- Exchange (123) - +- HashAggregate (122) - +- Project (121) - +- ShuffledHashJoin Inner BuildRight (120) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (92) - : : : : : +- Project (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Filter (94) - : : : : +- Scan parquet (93) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (119) - +- Filter (118) - +- Scan parquet (117) + Sort (132) + +- Exchange (131) + +- HashAggregate (130) + +- Exchange (129) + +- HashAggregate (128) + +- Project (127) + +- ShuffledHashJoin Inner BuildRight (126) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -124,536 +130,560 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(91) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(92) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(99) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(94) Filter +(100) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(95) Exchange +(101) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(97) Project +(103) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(98) Exchange +(104) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(101) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(103) Project +(109) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(104) Exchange +(110) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(107) Exchange +(113) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(109) Project +(115) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(110) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(113) Exchange +(119) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(121) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(116) Exchange +(122) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(119) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(127) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(122) HashAggregate +(128) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(123) Exchange +(129) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) HashAggregate +(130) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(125) Exchange +(131) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(132) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(127) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index c5d58658acd5..74f79bd3ee64 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index af8f35bc4b0e..66b2ccdc77b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,81 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- TakeOrderedAndProjectExecTransformer (55) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ ShuffledHashJoinExecTransformer Inner (43) - :- ^ InputIteratorTransformer (35) - : +- ^ InputAdapter (34) - : +- ^ ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- ^ ProjectExecTransformer (30) - : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : :- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ ShuffleQueryStage (26), Statistics(X) - : +- ColumnarExchange (25) - : +- ^ ProjectExecTransformer (23) - : +- ^ Scan parquet (22) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- ^ ProjectExecTransformer (37) - +- ^ Scan parquet (36) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- ShuffledHashJoin Inner BuildRight (77) - :- Exchange (73) - : +- Project (72) - : +- ShuffledHashJoin Inner BuildRight (71) - : :- Exchange (66) - : : +- Project (65) - : : +- ShuffledHashJoin Inner BuildRight (64) - : : :- Exchange (59) - : : : +- Filter (58) - : : : +- Scan parquet (57) - : : +- Exchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Exchange (70) - : +- Project (69) - : +- Filter (68) - : +- Scan parquet (67) - +- Exchange (76) - +- Filter (75) - +- Scan parquet (74) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -85,354 +89,370 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(55) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(56) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(57) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(58) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(59) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(61) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(62) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(63) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(65) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(66) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(68) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(69) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(70) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(72) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(73) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(75) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(76) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(79) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(80) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(82) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(83) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index e40294ba7679..4bd0783da6fe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,68 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ^ InputAdapter (43) - +- ^ ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ FilterExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ ProjectExecTransformer (32) - +- ^ FlushableHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- Filter (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- ShuffledHashJoin Inner BuildRight (61) - :- Exchange (56) - : +- Project (55) - : +- ShuffledHashJoin Inner BuildRight (54) - : :- Exchange (50) - : : +- Filter (49) - : : +- Scan parquet (48) - : +- Exchange (53) - : +- Filter (52) - : +- Scan parquet (51) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + Sort (71) + +- Exchange (70) + +- Filter (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- ShuffledHashJoin Inner BuildRight (64) + :- Exchange (59) + : +- Project (58) + : +- ShuffledHashJoin Inner BuildRight (57) + : :- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Exchange (56) + : +- Filter (55) + : +- Scan parquet (54) + +- Exchange (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) (1) Scan parquet @@ -72,548 +75,565 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [1]: [n_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(31) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(34) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(38) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(39) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(40) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(41) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(43) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(44) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(45) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(46) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(47) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(48) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(49) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(50) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(53) Exchange +(56) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(56) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Scan parquet +(60) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(58) Filter +(61) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(59) Project +(62) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) Exchange +(63) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(65) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(63) HashAggregate +(66) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(64) Exchange +(67) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(68) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(66) Filter +(69) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(67) Exchange +(70) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(71) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(69) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (116) +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (120) +- == Final Plan == - VeloxColumnarToRowExec (98) - +- ^ ProjectExecTransformer (96) - +- ^ RegularHashAggregateExecTransformer (95) - +- ^ RegularHashAggregateExecTransformer (94) - +- ^ ProjectExecTransformer (93) - +- ^ ShuffledHashJoinExecTransformer Inner (92) - :- ^ InputIteratorTransformer (87) - : +- ^ InputAdapter (86) - : +- ^ ShuffleQueryStage (85), Statistics(X) - : +- ColumnarExchange (84) - : +- ^ ProjectExecTransformer (82) - : +- ^ ShuffledHashJoinExecTransformer Inner (81) - : :- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ Scan parquet (70) - : +- ^ InputIteratorTransformer (80) - : +- ^ InputAdapter (79) - : +- ^ ShuffleQueryStage (78), Statistics(X) - : +- ReusedExchange (77) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ReusedExchange (88) + VeloxColumnarToRowExec (102) + +- ^ ProjectExecTransformer (100) + +- ^ RegularHashAggregateExecTransformer (99) + +- ^ RegularHashAggregateExecTransformer (98) + +- ^ ProjectExecTransformer (97) + +- ^ ShuffledHashJoinExecTransformer Inner (96) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (80) + : : +- ^ InputAdapter (79) + : : +- ^ ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ NoopFilter (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) + +- ^ InputIteratorTransformer (95) + +- ^ InputAdapter (94) + +- ^ ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == - HashAggregate (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) - - -(70) Scan parquet + HashAggregate (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildRight (109) + : :- Exchange (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (108) + : +- Filter (107) + : +- Scan parquet (106) + +- Exchange (115) + +- Project (114) + +- Filter (113) + +- Scan parquet (112) + + +(73) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(71) ProjectExecTransformer +(74) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(75) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(72) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(73) ColumnarExchange +(77) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(78) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(75) InputAdapter +(79) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) InputIteratorTransformer +(80) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(77) ReusedExchange [Reuses operator id: 11] +(81) ReusedExchange [Reuses operator id: 13] Output [2]: [s_suppkey#X, s_nationkey#X] -(78) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(79) InputAdapter +(83) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(80) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(81) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(82) ProjectExecTransformer +(86) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(83) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(88) ReusedExchange [Reuses operator id: 25] +(92) ReusedExchange [Reuses operator id: 28] Output [1]: [n_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [1]: [n_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [1]: [n_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(94) RegularHashAggregateExecTransformer +(98) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(95) RegularHashAggregateExecTransformer +(99) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(96) ProjectExecTransformer +(100) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(97) WholeStageCodegenTransformer (X) +(101) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(98) VeloxColumnarToRowExec +(102) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) Scan parquet +(103) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(100) Filter +(104) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(101) Exchange +(105) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(106) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(103) Filter +(107) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(104) Exchange +(108) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(106) Project +(110) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(107) Exchange +(111) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(110) Project +(114) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(115) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(117) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(118) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(115) HashAggregate +(119) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(116) AdaptiveSparkPlan +(120) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index a255925b847c..ec56c2f99543 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,48 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- HashAggregate (44) - +- Exchange (43) - +- HashAggregate (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin Inner BuildLeft (42) + :- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -52,200 +54,208 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(35) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(38) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(37) Filter +(39) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(38) Project +(40) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(39) Exchange +(41) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(41) Project +(43) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(42) HashAggregate +(44) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(43) Exchange +(45) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) HashAggregate +(46) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(45) Exchange +(47) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(48) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index eb454d4f7f23..f7d8f3c3b72f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftOuter BuildRight (42) - :- Exchange (37) - : +- Scan parquet (36) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftOuter BuildRight (43) + :- Exchange (38) + : +- Scan parquet (37) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -84,198 +85,202 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 0cd11b2c11f7..4f4492d881fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (32) - +- HashAggregate (31) - +- Project (30) - +- ShuffledHashJoin Inner BuildRight (29) - :- Exchange (25) - : +- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- Exchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- HashAggregate (33) + +- Project (32) + +- ShuffledHashJoin Inner BuildRight (31) + :- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -40,148 +42,156 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(23) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(24) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Scan parquet +(28) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) Exchange +(30) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) ShuffledHashJoin +(31) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(32) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(33) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 96f8900c810c..98249fae7dd7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- AQEShuffleRead (24) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner (19) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner (21) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- Project (37) - +- ShuffledHashJoin Inner BuildLeft (36) - :- Exchange (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Filter (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- Filter (30) - +- Scan parquet (29) + Sort (41) + +- Exchange (40) + +- Project (39) + +- ShuffledHashJoin Inner BuildLeft (38) + :- Exchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -47,328 +49,341 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(19) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(25) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(28) Exchange +(30) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(31) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(32) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(33) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(35) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(36) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(37) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(38) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (64) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ ProjectExecTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ Scan parquet (41) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(41) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(42) ProjectExecTransformer +(44) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(51) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(53) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(54) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(55) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(56) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(57) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(58) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(60) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(63) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index d8bc92e439c3..230e9c890d1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FlushableHashAggregateExecTransformer (25) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ^ InputAdapter (38) + +- ^ ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- ShuffledHashJoin Inner BuildRight (52) - :- Exchange (48) - : +- BroadcastHashJoin LeftAnti BuildRight (47) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Exchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (63) + +- Exchange (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -66,274 +68,282 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(28) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(30) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(33) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(34) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(36) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(39) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(40) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(43) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(44) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(45) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(46) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(48) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Scan parquet +(51) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(52) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) Exchange +(53) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(53) Project +(55) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(56) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(58) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(63) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index a76baac13557..3848cd252b45 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ ShuffledHashJoinExecTransformer Inner (28) - :- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ FilterExecTransformer (27) - +- ^ ProjectExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ Scan parquet (17) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (53) - +- HashAggregate (52) - +- Project (51) - +- ShuffledHashJoin Inner BuildRight (50) - :- Project (43) - : +- ShuffledHashJoin Inner BuildRight (42) - : :- Exchange (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Filter (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Filter (45) - +- Scan parquet (44) + HashAggregate (56) + +- HashAggregate (55) + +- Project (54) + +- ShuffledHashJoin Inner BuildRight (53) + :- Project (46) + : +- ShuffledHashJoin Inner BuildRight (45) + : :- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Exchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -60,250 +63,262 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_container#X] + +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) NoopFilter +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: [l_partkey#X, l_quantity#X] + +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(20) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(21) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(30) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(37) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(40) Project +(43) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(41) Exchange +(44) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(43) Project +(46) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(45) Filter +(48) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(46) HashAggregate +(49) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(47) Exchange +(50) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(49) Filter +(52) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(50) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(51) Project +(54) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) HashAggregate +(55) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(53) HashAggregate +(56) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(54) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index 36c934dd0416..08b902eed5ed 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,93 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- TakeOrderedAndProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ ShuffledHashJoinExecTransformer Inner (55) - :- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ColumnarExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ ShuffledHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ ShuffleQueryStage (30), Statistics(X) - : +- ColumnarExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) - : :- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ ProjectExecTransformer (25) - : +- ^ FilterExecTransformer (24) - : +- ^ RegularHashAggregateExecTransformer (23) - : +- ^ InputIteratorTransformer (22) - : +- ^ InputAdapter (21) - : +- ^ ShuffleQueryStage (20), Statistics(X) - : +- ColumnarExchange (19) - : +- ^ ProjectExecTransformer (17) - : +- ^ FlushableHashAggregateExecTransformer (16) - : +- ^ Scan parquet (15) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) - :- ^ InputIteratorTransformer (46) - : +- ^ InputAdapter (45) - : +- ^ ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ Scan parquet (40) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ReusedExchange (47) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- HashAggregate (91) - +- Project (90) - +- ShuffledHashJoin Inner BuildRight (89) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildLeft (76) - : :- Exchange (64) - : : +- Filter (63) - : : +- Scan parquet (62) - : +- Exchange (75) - : +- ShuffledHashJoin LeftSemi BuildRight (74) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Project (73) - : +- Filter (72) - : +- HashAggregate (71) - : +- Exchange (70) - : +- HashAggregate (69) - : +- Scan parquet (68) - +- ShuffledHashJoin LeftSemi BuildRight (88) - :- Exchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- Project (87) - +- Filter (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Scan parquet (82) + TakeOrderedAndProject (96) + +- HashAggregate (95) + +- HashAggregate (94) + +- Project (93) + +- ShuffledHashJoin Inner BuildRight (92) + :- Exchange (81) + : +- Project (80) + : +- ShuffledHashJoin Inner BuildLeft (79) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (78) + : +- ShuffledHashJoin LeftSemi BuildRight (77) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Project (76) + : +- Filter (75) + : +- HashAggregate (74) + : +- Exchange (73) + : +- HashAggregate (72) + : +- Scan parquet (71) + +- ShuffledHashJoin LeftSemi BuildRight (91) + :- Exchange (84) + : +- Filter (83) + : +- Scan parquet (82) + +- Project (90) + +- Filter (89) + +- HashAggregate (88) + +- Exchange (87) + +- HashAggregate (86) + +- Scan parquet (85) (1) Scan parquet @@ -97,420 +100,432 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(19) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(21) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(23) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(24) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(25) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(26) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(29) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(34) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(36) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(38) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(39) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(40) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(47) ReusedExchange [Reuses operator id: 19] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(48) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(52) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(54) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(57) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(58) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(59) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(60) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(61) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(62) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(63) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(64) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(68) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(69) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(67) Exchange +(70) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(75) Exchange +(78) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(79) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(77) Project +(80) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(78) Exchange +(81) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(81) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(83) HashAggregate +(86) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(84) Exchange +(87) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(88) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(86) Filter +(89) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(87) Project +(90) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(88) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(89) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(90) Project +(93) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(91) HashAggregate +(94) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(92) HashAggregate +(95) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(93) TakeOrderedAndProject +(96) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(94) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index abcddb7ca47a..29b5d69fa0e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (31) - +- HashAggregate (30) - +- Project (29) - +- ShuffledHashJoin Inner BuildRight (28) - :- Exchange (24) - : +- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- Exchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- HashAggregate (32) + +- Project (31) + +- ShuffledHashJoin Inner BuildRight (30) + :- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Exchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -39,144 +41,152 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(21) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(22) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(23) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(24) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) Scan parquet +(27) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) Exchange +(29) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) ShuffledHashJoin +(30) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(31) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(32) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index e24568ef1f6c..c5dc33a694ee 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,114 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- AQEShuffleRead (77) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ ShuffledHashJoinExecTransformer Inner (72) - :- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ ShuffleQueryStage (62), Statistics(X) - : +- ColumnarExchange (61) - : +- ^ ProjectExecTransformer (59) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : +- ColumnarExchange (25) - : : +- ^ ProjectExecTransformer (23) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : :- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : +- ColumnarExchange (11) - : : : +- ^ ProjectExecTransformer (9) - : : : +- ^ Scan parquet (8) - : : +- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ Scan parquet (15) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) - : :- ^ InputIteratorTransformer (35) - : : +- ^ InputAdapter (34) - : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ Scan parquet (29) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ReusedExchange (36) - +- ^ InputIteratorTransformer (71) - +- ^ InputAdapter (70) - +- ^ ShuffleQueryStage (69), Statistics(X) - +- ColumnarExchange (68) - +- ^ ProjectExecTransformer (66) - +- ^ Scan parquet (65) + VeloxColumnarToRowExec (83) + +- AQEShuffleRead (82) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (31) + : : +- ^ InputAdapter (30) + : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : :- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (117) - +- Exchange (116) - +- Project (115) - +- ShuffledHashJoin Inner BuildRight (114) - :- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- Exchange (106) - : +- Project (105) - : +- ShuffledHashJoin Inner BuildLeft (104) - : :- Exchange (90) - : : +- ShuffledHashJoin LeftSemi BuildRight (89) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (88) - : : +- Project (87) - : : +- Filter (86) - : : +- Scan parquet (85) - : +- Exchange (103) - : +- Filter (102) - : +- HashAggregate (101) - : +- HashAggregate (100) - : +- ShuffledHashJoin LeftSemi BuildRight (99) - : :- Exchange (94) - : : +- Project (93) - : : +- Filter (92) - : : +- Scan parquet (91) - : +- Exchange (98) - : +- Project (97) - : +- Filter (96) - : +- Scan parquet (95) - +- Exchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (122) + +- Exchange (121) + +- Project (120) + +- ShuffledHashJoin Inner BuildRight (119) + :- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin LeftSemi BuildRight (112) + : :- Exchange (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildLeft (109) + : :- Exchange (95) + : : +- ShuffledHashJoin LeftSemi BuildRight (94) + : : :- Exchange (89) + : : : +- Filter (88) + : : : +- Scan parquet (87) + : : +- Exchange (93) + : : +- Project (92) + : : +- Filter (91) + : : +- Scan parquet (90) + : +- Exchange (108) + : +- Filter (107) + : +- HashAggregate (106) + : +- HashAggregate (105) + : +- ShuffledHashJoin LeftSemi BuildRight (104) + : :- Exchange (99) + : : +- Project (98) + : : +- Filter (97) + : : +- Scan parquet (96) + : +- Exchange (103) + : +- Project (102) + : +- Filter (101) + : +- Scan parquet (100) + +- Exchange (118) + +- Project (117) + +- Filter (116) + +- Scan parquet (115) (1) Scan parquet @@ -118,500 +123,520 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(16) ProjectExecTransformer +(18) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(17) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(18) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(23) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(29) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(30) ProjectExecTransformer +(33) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(31) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(32) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(34) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(35) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(36) ReusedExchange [Reuses operator id: 18] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(37) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [p_partkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -(40) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(58) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(59) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(60) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(61) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(64) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(65) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(66) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(67) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(68) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(69) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(74) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(75) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(77) AQEShuffleRead +(82) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(78) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(79) Scan parquet +(84) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(85) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(81) Exchange +(86) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(84) Exchange +(89) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(92) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) Exchange +(93) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) Exchange +(95) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(92) Filter +(97) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(93) Project +(98) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(97) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(100) HashAggregate +(105) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(101) HashAggregate +(106) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(102) Filter +(107) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(103) Exchange +(108) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(105) Project +(110) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(106) Exchange +(111) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(108) Project +(113) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(109) Exchange +(114) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(112) Project +(117) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(113) Exchange +(118) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(120) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(116) Exchange +(121) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(122) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(118) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index fec143f3c82c..91bb400aac04 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,108 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (113) +AdaptiveSparkPlan (118) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ RegularHashAggregateExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ FlushableHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner (65) - :- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (43) - : : +- ^ InputAdapter (42) - : : +- ^ ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer Inner (37) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (36) - : : +- ^ InputAdapter (35) - : : +- ^ ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- ^ ProjectExecTransformer (31) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : : :- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (21) - : : : +- ^ InputAdapter (20) - : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : +- ColumnarExchange (18) - : : : +- ^ ProjectExecTransformer (16) - : : : +- ^ Scan parquet (15) - : : +- ^ InputIteratorTransformer (29) - : : +- ^ InputAdapter (28) - : : +- ^ ShuffleQueryStage (27), Statistics(X) - : : +- ColumnarExchange (26) - : : +- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ Scan parquet (44) - +- ^ InputIteratorTransformer (64) - +- ^ InputAdapter (63) - +- ^ ShuffleQueryStage (62), Statistics(X) - +- ColumnarExchange (61) - +- ^ ProjectExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (112) - +- HashAggregate (111) - +- Exchange (110) - +- HashAggregate (109) - +- Project (108) - +- ShuffledHashJoin Inner BuildRight (107) - :- Exchange (102) - : +- Project (101) - : +- ShuffledHashJoin Inner BuildRight (100) - : :- Exchange (95) - : : +- Project (94) - : : +- ShuffledHashJoin Inner BuildLeft (93) - : : :- Exchange (79) - : : : +- Filter (78) - : : : +- Scan parquet (77) - : : +- Exchange (92) - : : +- ShuffledHashJoin LeftAnti BuildRight (91) - : : :- ShuffledHashJoin LeftSemi BuildRight (86) - : : : :- Exchange (83) - : : : : +- Project (82) - : : : : +- Filter (81) - : : : : +- Scan parquet (80) - : : : +- Exchange (85) - : : : +- Scan parquet (84) - : : +- Exchange (90) - : : +- Project (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (99) - : +- Project (98) - : +- Filter (97) - : +- Scan parquet (96) - +- Exchange (106) - +- Project (105) - +- Filter (104) - +- Scan parquet (103) + TakeOrderedAndProject (117) + +- HashAggregate (116) + +- Exchange (115) + +- HashAggregate (114) + +- Project (113) + +- ShuffledHashJoin Inner BuildRight (112) + :- Exchange (107) + : +- Project (106) + : +- ShuffledHashJoin Inner BuildRight (105) + : :- Exchange (100) + : : +- Project (99) + : : +- ShuffledHashJoin Inner BuildLeft (98) + : : :- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Exchange (97) + : : +- ShuffledHashJoin LeftAnti BuildRight (96) + : : :- ShuffledHashJoin LeftSemi BuildRight (91) + : : : :- Exchange (88) + : : : : +- Project (87) + : : : : +- Filter (86) + : : : : +- Scan parquet (85) + : : : +- Exchange (90) + : : : +- Scan parquet (89) + : : +- Exchange (95) + : : +- Project (94) + : : +- Filter (93) + : : +- Scan parquet (92) + : +- Exchange (104) + : +- Project (103) + : +- Filter (102) + : +- Scan parquet (101) + +- Exchange (111) + +- Project (110) + +- Filter (109) + +- Scan parquet (108) (1) Scan parquet @@ -112,482 +117,502 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(22) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(23) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(24) ProjectExecTransformer +(26) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(25) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(26) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(28) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(29) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(30) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(31) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(33) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(35) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(36) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(37) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(38) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(40) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(42) InputAdapter +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(43) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(45) ProjectExecTransformer +(48) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(58) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(59) ProjectExecTransformer +(63) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(61) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(63) InputAdapter +(68) InputAdapter Input [1]: [n_nationkey#X] -(64) InputIteratorTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -(65) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(66) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(67) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(68) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(69) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(70) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(72) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(73) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(74) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(75) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(76) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(77) Scan parquet +(82) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(78) Filter +(83) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(79) Exchange +(84) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) Scan parquet +(85) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(86) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(82) Project +(87) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(83) Exchange +(88) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(89) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(87) Scan parquet +(92) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(88) Filter +(93) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(89) Project +(94) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(94) Project +(99) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(95) Exchange +(100) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(101) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(97) Filter +(102) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(98) Project +(103) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(99) Exchange +(104) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(101) Project +(106) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(102) Exchange +(107) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(104) Filter +(109) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(105) Project +(110) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(106) Exchange +(111) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(108) Project +(113) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(109) HashAggregate +(114) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(110) Exchange +(115) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) HashAggregate +(116) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(112) TakeOrderedAndProject +(117) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(113) AdaptiveSparkPlan +(118) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 2f5f9d303931..ba18df0d0119 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ^ InputAdapter (29) + +- ^ ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- ShuffledHashJoin LeftAnti BuildRight (38) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (37) - +- Scan parquet (36) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- ShuffledHashJoin LeftAnti BuildRight (39) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Exchange (38) + +- Scan parquet (37) (1) Scan parquet @@ -50,300 +51,309 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(17) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(20) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(35) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(39) Project +(40) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(40) HashAggregate +(41) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(43) Exchange +(44) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (65) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ColumnarExchange (50) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == - HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(46) Scan parquet +(47) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(47) ProjectExecTransformer +(48) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(49) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(48) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(49) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(50) ColumnarExchange +(52) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(52) InputAdapter +(54) InputAdapter Input [2]: [sum#X, count#X] -(53) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [sum#X, count#X] -(54) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(56) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(57) Scan parquet +(59) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(58) Filter +(60) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(59) Project +(61) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(60) HashAggregate +(62) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(61) Exchange +(63) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(63) AdaptiveSparkPlan +(65) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 48b0e5f28558..08588f64d24f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,57 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- TakeOrderedAndProjectExecTransformer (35) - +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Project (52) - +- ShuffledHashJoin Inner BuildRight (51) - :- Exchange (46) - : +- Project (45) - : +- ShuffledHashJoin Inner BuildLeft (44) - : :- Exchange (40) - : : +- Project (39) - : : +- Filter (38) - : : +- Scan parquet (37) - : +- Exchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Exchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -61,244 +64,256 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(32) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(33) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(37) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(38) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(39) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(40) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(43) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(45) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(46) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(49) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(52) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(53) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(54) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(55) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 33b40f964185..421f6a412ec7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,49 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- ShuffledHashJoin LeftSemi BuildRight (41) - :- Exchange (36) - : +- Project (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftSemi BuildRight (43) + :- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -53,204 +55,212 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(35) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(36) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(39) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(38) Filter +(40) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(39) Project +(41) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(40) Exchange +(42) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) Project +(44) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(43) HashAggregate +(45) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(44) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(46) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 719d3611db67..50173820058c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,121 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (134) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- ShuffledHashJoin Inner BuildRight (127) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (126) + +- Project (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -125,540 +131,564 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [1]: [r_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(91) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(93) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(94) Project +(100) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Exchange +(101) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(97) Project +(103) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(98) Exchange +(104) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(101) Exchange +(107) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(103) Project +(109) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(110) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(109) Project +(115) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(116) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(113) Exchange +(119) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(121) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(116) Exchange +(122) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(119) Project +(125) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(120) Exchange +(126) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(127) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(122) Project +(128) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(123) HashAggregate +(129) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(124) Exchange +(130) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(131) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(126) Exchange +(132) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(133) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(128) AdaptiveSparkPlan +(134) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index 2e51f22a6b0e..51c5836bdd11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index eca3692340aa..67f4274b3aa3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,117 +1,122 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ^ InputAdapter (81) - +- ^ ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- ^ RegularHashAggregateExecTransformer (77) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FlushableHashAggregateExecTransformer (70) - +- ^ ProjectExecTransformer (69) - +- ^ ShuffledHashJoinExecTransformer Inner (68) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65), Statistics(X) - +- ReusedExchange (64) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ^ InputAdapter (86) + +- ^ ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner (73) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ^ InputAdapter (71) + +- ^ ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- HashAggregate (120) - +- Exchange (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin Inner BuildRight (110) - : :- Exchange (106) - : : +- Project (105) - : : +- ShuffledHashJoin Inner BuildRight (104) - : : :- Exchange (100) - : : : +- Project (99) - : : : +- ShuffledHashJoin Inner BuildRight (98) - : : : :- Exchange (94) - : : : : +- Project (93) - : : : : +- ShuffledHashJoin Inner BuildLeft (92) - : : : : :- Exchange (88) - : : : : : +- Filter (87) - : : : : : +- Scan parquet (86) - : : : : +- Exchange (91) - : : : : +- Filter (90) - : : : : +- Scan parquet (89) - : : : +- Exchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- Exchange (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (109) - : +- Filter (108) - : +- Scan parquet (107) - +- Exchange (115) - +- Filter (114) - +- Scan parquet (113) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildRight (109) + : : :- Exchange (105) + : : : +- Project (104) + : : : +- ShuffledHashJoin Inner BuildRight (103) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- ShuffledHashJoin Inner BuildLeft (97) + : : : : :- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,516 +126,536 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(64) ReusedExchange [Reuses operator id: 53] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(65) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(66) InputAdapter +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(67) InputIteratorTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(68) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(69) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(70) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(71) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(72) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(73) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(75) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(78) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(79) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(81) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(82) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(83) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(85) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(86) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(94) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(95) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) Exchange +(96) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(93) Project +(98) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(97) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(99) Project +(104) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(100) Exchange +(105) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(102) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(103) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(105) Project +(110) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(106) Exchange +(111) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(108) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(109) Exchange +(114) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(111) Project +(116) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(112) Exchange +(117) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(115) Exchange +(120) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(117) Project +(122) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(118) HashAggregate +(123) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(119) Exchange +(124) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) HashAggregate +(125) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(121) Exchange +(126) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(127) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 1883d0df585f..16838a0513b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,158 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (169) +AdaptiveSparkPlan (177) +- == Final Plan == - VeloxColumnarToRowExec (117) - +- ^ SortExecTransformer (115) - +- ^ InputIteratorTransformer (114) - +- ^ InputAdapter (113) - +- ^ ShuffleQueryStage (112), Statistics(X) - +- ColumnarExchange (111) - +- ^ ProjectExecTransformer (109) - +- ^ RegularHashAggregateExecTransformer (108) - +- ^ InputIteratorTransformer (107) - +- ^ InputAdapter (106) - +- ^ ShuffleQueryStage (105), Statistics(X) - +- ColumnarExchange (104) - +- ^ ProjectExecTransformer (102) - +- ^ FlushableHashAggregateExecTransformer (101) - +- ^ ProjectExecTransformer (100) - +- ^ ShuffledHashJoinExecTransformer Inner (99) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (77) - : : +- ^ InputAdapter (76) - : : +- ^ ShuffleQueryStage (75), Statistics(X) - : : +- ColumnarExchange (74) - : : +- ^ ProjectExecTransformer (72) - : : +- ^ ShuffledHashJoinExecTransformer Inner (71) - : : :- ^ InputIteratorTransformer (63) - : : : +- ^ InputAdapter (62) - : : : +- ^ ShuffleQueryStage (61), Statistics(X) - : : : +- ColumnarExchange (60) - : : : +- ^ ProjectExecTransformer (58) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : : : :- ^ InputIteratorTransformer (49) - : : : : +- ^ InputAdapter (48) - : : : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : : : +- ColumnarExchange (46) - : : : : +- ^ ProjectExecTransformer (44) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : : : :- ^ InputIteratorTransformer (35) - : : : : : +- ^ InputAdapter (34) - : : : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- ^ ProjectExecTransformer (30) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : : : :- ^ InputIteratorTransformer (21) - : : : : : : +- ^ InputAdapter (20) - : : : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : : : +- ColumnarExchange (18) - : : : : : : +- ^ ProjectExecTransformer (16) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (14) - : : : : : : +- ^ InputAdapter (13) - : : : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : : : +- ColumnarExchange (11) - : : : : : : +- ^ ProjectExecTransformer (9) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (28) - : : : : : +- ^ InputAdapter (27) - : : : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : : : +- ColumnarExchange (25) - : : : : : +- ^ ProjectExecTransformer (23) - : : : : : +- ^ Scan parquet (22) - : : : : +- ^ InputIteratorTransformer (42) - : : : : +- ^ InputAdapter (41) - : : : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : : : +- ColumnarExchange (39) - : : : : +- ^ ProjectExecTransformer (37) - : : : : +- ^ Scan parquet (36) - : : : +- ^ InputIteratorTransformer (56) - : : : +- ^ InputAdapter (55) - : : : +- ^ ShuffleQueryStage (54), Statistics(X) - : : : +- ColumnarExchange (53) - : : : +- ^ ProjectExecTransformer (51) - : : : +- ^ Scan parquet (50) - : : +- ^ InputIteratorTransformer (70) - : : +- ^ InputAdapter (69) - : : +- ^ ShuffleQueryStage (68), Statistics(X) - : : +- ColumnarExchange (67) - : : +- ^ ProjectExecTransformer (65) - : : +- ^ Scan parquet (64) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ColumnarExchange (81) - : +- ^ ProjectExecTransformer (79) - : +- ^ Scan parquet (78) - +- ^ InputIteratorTransformer (98) - +- ^ InputAdapter (97) - +- ^ ShuffleQueryStage (96), Statistics(X) - +- ColumnarExchange (95) - +- ^ ProjectExecTransformer (93) - +- ^ Scan parquet (92) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ^ InputAdapter (121) + +- ^ ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ^ InputAdapter (114) + +- ^ ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner (107) + :- ^ InputIteratorTransformer (98) + : +- ^ InputAdapter (97) + : +- ^ ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : :- ^ InputIteratorTransformer (83) + : : +- ^ InputAdapter (82) + : : +- ^ ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ^ InputAdapter (67) + : : : +- ^ ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ^ InputAdapter (52) + : : : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ^ InputAdapter (37) + : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ^ InputAdapter (22) + : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ^ InputAdapter (15) + : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ^ InputAdapter (30) + : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ^ InputAdapter (45) + : : : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ^ InputAdapter (60) + : : : +- ^ ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ^ InputAdapter (105) + +- ^ ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (168) - +- Exchange (167) - +- HashAggregate (166) - +- Exchange (165) - +- HashAggregate (164) - +- Project (163) - +- ShuffledHashJoin Inner BuildRight (162) - :- Exchange (157) - : +- Project (156) - : +- ShuffledHashJoin Inner BuildRight (155) - : :- Exchange (151) - : : +- Project (150) - : : +- ShuffledHashJoin Inner BuildRight (149) - : : :- Exchange (145) - : : : +- Project (144) - : : : +- ShuffledHashJoin Inner BuildRight (143) - : : : :- Exchange (139) - : : : : +- Project (138) - : : : : +- ShuffledHashJoin Inner BuildRight (137) - : : : : :- Exchange (133) - : : : : : +- Project (132) - : : : : : +- ShuffledHashJoin Inner BuildRight (131) - : : : : : :- Exchange (127) - : : : : : : +- Project (126) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) - : : : : : : :- Exchange (121) - : : : : : : : +- Project (120) - : : : : : : : +- Filter (119) - : : : : : : : +- Scan parquet (118) - : : : : : : +- Exchange (124) - : : : : : : +- Filter (123) - : : : : : : +- Scan parquet (122) - : : : : : +- Exchange (130) - : : : : : +- Filter (129) - : : : : : +- Scan parquet (128) - : : : : +- Exchange (136) - : : : : +- Filter (135) - : : : : +- Scan parquet (134) - : : : +- Exchange (142) - : : : +- Filter (141) - : : : +- Scan parquet (140) - : : +- Exchange (148) - : : +- Filter (147) - : : +- Scan parquet (146) - : +- Exchange (154) - : +- Filter (153) - : +- Scan parquet (152) - +- Exchange (161) - +- Project (160) - +- Filter (159) - +- Scan parquet (158) + Sort (176) + +- Exchange (175) + +- HashAggregate (174) + +- Exchange (173) + +- HashAggregate (172) + +- Project (171) + +- ShuffledHashJoin Inner BuildRight (170) + :- Exchange (165) + : +- Project (164) + : +- ShuffledHashJoin Inner BuildRight (163) + : :- Exchange (159) + : : +- Project (158) + : : +- ShuffledHashJoin Inner BuildRight (157) + : : :- Exchange (153) + : : : +- Project (152) + : : : +- ShuffledHashJoin Inner BuildRight (151) + : : : :- Exchange (147) + : : : : +- Project (146) + : : : : +- ShuffledHashJoin Inner BuildRight (145) + : : : : :- Exchange (141) + : : : : : +- Project (140) + : : : : : +- ShuffledHashJoin Inner BuildRight (139) + : : : : : :- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) + : : : : : : :- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Exchange (132) + : : : : : : +- Filter (131) + : : : : : : +- Scan parquet (130) + : : : : : +- Exchange (138) + : : : : : +- Filter (137) + : : : : : +- Scan parquet (136) + : : : : +- Exchange (144) + : : : : +- Filter (143) + : : : : +- Scan parquet (142) + : : : +- Exchange (150) + : : : +- Filter (149) + : : : +- Scan parquet (148) + : : +- Exchange (156) + : : +- Filter (155) + : : +- Scan parquet (154) + : +- Exchange (162) + : +- Filter (161) + : +- Scan parquet (160) + +- Exchange (169) + +- Project (168) + +- Filter (167) + +- Scan parquet (166) (1) Scan parquet @@ -162,712 +170,744 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(73) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(74) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(76) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(77) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) ProjectExecTransformer +(85) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(80) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(81) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(92) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(93) ProjectExecTransformer +(100) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(94) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(95) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(97) InputAdapter +(105) InputAdapter Input [1]: [r_regionkey#X] -(98) InputIteratorTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -(99) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(101) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(102) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(103) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(104) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(105) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(106) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(107) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(108) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(109) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(110) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(111) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(112) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(113) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(114) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(115) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(116) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(117) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(118) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(120) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(121) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(130) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(123) Filter +(131) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(124) Exchange +(132) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(133) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(126) Project +(134) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(127) Exchange +(135) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(129) Filter +(137) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(130) Exchange +(138) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(132) Project +(140) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(133) Exchange +(141) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(142) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(135) Filter +(143) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(136) Exchange +(144) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(145) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(138) Project +(146) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(139) Exchange +(147) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Scan parquet +(148) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(141) Filter +(149) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(142) Exchange +(150) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) ShuffledHashJoin +(151) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(144) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(145) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(154) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(147) Filter +(155) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(148) Exchange +(156) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(157) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(150) Project +(158) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(151) Exchange +(159) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(160) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(153) Filter +(161) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(154) Exchange +(162) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(163) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(156) Project +(164) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(157) Exchange +(165) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(166) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(159) Filter +(167) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(160) Project +(168) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(161) Exchange +(169) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(162) ShuffledHashJoin +(170) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(163) Project +(171) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(164) HashAggregate +(172) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(165) Exchange +(173) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) HashAggregate +(174) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(167) Exchange +(175) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(176) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(169) AdaptiveSparkPlan +(177) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index feffe6710113..a24b228c5f77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,120 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (126) - +- Exchange (125) - +- HashAggregate (124) - +- Exchange (123) - +- HashAggregate (122) - +- Project (121) - +- ShuffledHashJoin Inner BuildRight (120) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (92) - : : : : : +- Project (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Filter (94) - : : : : +- Scan parquet (93) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (119) - +- Filter (118) - +- Scan parquet (117) + Sort (132) + +- Exchange (131) + +- HashAggregate (130) + +- Exchange (129) + +- HashAggregate (128) + +- Project (127) + +- ShuffledHashJoin Inner BuildRight (126) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -124,536 +130,560 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(91) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(92) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(99) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(94) Filter +(100) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(95) Exchange +(101) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(97) Project +(103) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(98) Exchange +(104) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(101) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(103) Project +(109) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(104) Exchange +(110) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(107) Exchange +(113) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(109) Project +(115) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(110) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(113) Exchange +(119) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(121) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(116) Exchange +(122) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(119) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(127) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(122) HashAggregate +(128) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(123) Exchange +(129) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) HashAggregate +(130) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(125) Exchange +(131) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(132) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(127) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index 0217849e5de2..41613c36f7bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,30 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (27) +AdaptiveSparkPlan (28) +- == Final Plan == - VeloxColumnarToRowExec (18) - +- ^ SortExecTransformer (16) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (19) + +- ^ SortExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ^ InputAdapter (9) + +- ^ ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (26) - +- Exchange (25) - +- HashAggregate (24) - +- Exchange (23) - +- HashAggregate (22) - +- Project (21) - +- Filter (20) - +- Scan parquet (19) + Sort (27) + +- Exchange (26) + +- HashAggregate (25) + +- Exchange (24) + +- HashAggregate (23) + +- Project (22) + +- Filter (21) + +- Scan parquet (20) (1) Scan parquet @@ -34,116 +35,120 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(4) ProjectExecTransformer +(5) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) WholeStageCodegenTransformer (X) +(6) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(6) ColumnarExchange +(7) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(12) ColumnarExchange +(13) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(14) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(14) InputAdapter +(15) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(15) InputIteratorTransformer +(16) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) SortExecTransformer +(17) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(17) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(18) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(19) Scan parquet +(20) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(20) Filter +(21) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(21) Project +(22) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(22) HashAggregate +(23) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(23) Exchange +(24) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(24) HashAggregate +(25) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(25) Exchange +(26) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Sort +(27) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(27) AdaptiveSparkPlan +(28) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index d88b2d72aa24..79ff600fa6b7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,81 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (83) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- TakeOrderedAndProjectExecTransformer (55) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ ShuffledHashJoinExecTransformer Inner (43) - :- ^ InputIteratorTransformer (35) - : +- ^ InputAdapter (34) - : +- ^ ShuffleQueryStage (33), Statistics(X) - : +- ColumnarExchange (32) - : +- ^ ProjectExecTransformer (30) - : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : :- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (28) - : +- ^ InputAdapter (27) - : +- ^ ShuffleQueryStage (26), Statistics(X) - : +- ColumnarExchange (25) - : +- ^ ProjectExecTransformer (23) - : +- ^ Scan parquet (22) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ ShuffleQueryStage (40), Statistics(X) - +- ColumnarExchange (39) - +- ^ ProjectExecTransformer (37) - +- ^ Scan parquet (36) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (82) - +- HashAggregate (81) - +- Exchange (80) - +- HashAggregate (79) - +- Project (78) - +- ShuffledHashJoin Inner BuildRight (77) - :- Exchange (73) - : +- Project (72) - : +- ShuffledHashJoin Inner BuildRight (71) - : :- Exchange (66) - : : +- Project (65) - : : +- ShuffledHashJoin Inner BuildRight (64) - : : :- Exchange (59) - : : : +- Filter (58) - : : : +- Scan parquet (57) - : : +- Exchange (63) - : : +- Project (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- Exchange (70) - : +- Project (69) - : +- Filter (68) - : +- Scan parquet (67) - +- Exchange (76) - +- Filter (75) - +- Scan parquet (74) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -85,360 +89,376 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] + +(3) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(55) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(56) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(57) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(58) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(59) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(61) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(62) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(63) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(65) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(66) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(68) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(69) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(70) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(72) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(73) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(75) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(76) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(79) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(80) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(82) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(83) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index 486fe217ad6f..1b5a8743db5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,68 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (47) - +- ^ SortExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ^ InputAdapter (43) - +- ^ ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ FilterExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ ProjectExecTransformer (32) - +- ^ FlushableHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ^ InputAdapter (46) + +- ^ ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- Filter (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- ShuffledHashJoin Inner BuildRight (61) - :- Exchange (56) - : +- Project (55) - : +- ShuffledHashJoin Inner BuildRight (54) - : :- Exchange (50) - : : +- Filter (49) - : : +- Scan parquet (48) - : +- Exchange (53) - : +- Filter (52) - : +- Scan parquet (51) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + Sort (71) + +- Exchange (70) + +- Filter (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- ShuffledHashJoin Inner BuildRight (64) + :- Exchange (59) + : +- Project (58) + : +- ShuffledHashJoin Inner BuildRight (57) + : :- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Exchange (56) + : +- Filter (55) + : +- Scan parquet (54) + +- Exchange (63) + +- Project (62) + +- Filter (61) + +- Scan parquet (60) (1) Scan parquet @@ -72,556 +75,573 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(3) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [1]: [n_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(31) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(34) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(36) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(37) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(38) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(39) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(40) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(41) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(43) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(44) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(45) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(46) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(47) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(48) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(49) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(50) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(53) Exchange +(56) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(55) Project +(58) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(56) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Scan parquet +(60) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(58) Filter +(61) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(59) Project +(62) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) Exchange +(63) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(65) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(63) HashAggregate +(66) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(64) Exchange +(67) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(68) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(66) Filter +(69) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(67) Exchange +(70) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(71) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(69) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (116) +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (120) +- == Final Plan == - VeloxColumnarToRowExec (98) - +- ^ ProjectExecTransformer (96) - +- ^ RegularHashAggregateExecTransformer (95) - +- ^ RegularHashAggregateExecTransformer (94) - +- ^ ProjectExecTransformer (93) - +- ^ ShuffledHashJoinExecTransformer Inner (92) - :- ^ InputIteratorTransformer (87) - : +- ^ InputAdapter (86) - : +- ^ ShuffleQueryStage (85), Statistics(X) - : +- ColumnarExchange (84) - : +- ^ ProjectExecTransformer (82) - : +- ^ ShuffledHashJoinExecTransformer Inner (81) - : :- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ Scan parquet (70) - : +- ^ InputIteratorTransformer (80) - : +- ^ InputAdapter (79) - : +- ^ ShuffleQueryStage (78), Statistics(X) - : +- ReusedExchange (77) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ReusedExchange (88) + VeloxColumnarToRowExec (102) + +- ^ ProjectExecTransformer (100) + +- ^ RegularHashAggregateExecTransformer (99) + +- ^ RegularHashAggregateExecTransformer (98) + +- ^ ProjectExecTransformer (97) + +- ^ ShuffledHashJoinExecTransformer Inner (96) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (80) + : : +- ^ InputAdapter (79) + : : +- ^ ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ NoopFilter (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) + +- ^ InputIteratorTransformer (95) + +- ^ InputAdapter (94) + +- ^ ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == - HashAggregate (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) - - -(70) Scan parquet + HashAggregate (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildRight (109) + : :- Exchange (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (108) + : +- Filter (107) + : +- Scan parquet (106) + +- Exchange (115) + +- Project (114) + +- Filter (113) + +- Scan parquet (112) + + +(73) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(71) ProjectExecTransformer +(74) NoopFilter +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] + +(75) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(72) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(73) ColumnarExchange +(77) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(78) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(75) InputAdapter +(79) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) InputIteratorTransformer +(80) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(77) ReusedExchange [Reuses operator id: 11] +(81) ReusedExchange [Reuses operator id: 13] Output [2]: [s_suppkey#X, s_nationkey#X] -(78) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(79) InputAdapter +(83) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(80) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(81) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(82) ProjectExecTransformer +(86) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(83) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(88) ReusedExchange [Reuses operator id: 25] +(92) ReusedExchange [Reuses operator id: 28] Output [1]: [n_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [1]: [n_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [1]: [n_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(94) RegularHashAggregateExecTransformer +(98) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(95) RegularHashAggregateExecTransformer +(99) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(96) ProjectExecTransformer +(100) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(97) WholeStageCodegenTransformer (X) +(101) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(98) VeloxColumnarToRowExec +(102) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) Scan parquet +(103) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(100) Filter +(104) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(101) Exchange +(105) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(106) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(103) Filter +(107) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(104) Exchange +(108) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(106) Project +(110) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(107) Exchange +(111) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(110) Project +(114) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(115) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(117) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(118) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(115) HashAggregate +(119) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(116) AdaptiveSparkPlan +(120) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index d77bf02f980a..595d52af5256 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,48 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (47) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (46) - +- Exchange (45) - +- HashAggregate (44) - +- Exchange (43) - +- HashAggregate (42) - +- Project (41) - +- ShuffledHashJoin Inner BuildLeft (40) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (39) - +- Project (38) - +- Filter (37) - +- Scan parquet (36) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin Inner BuildLeft (42) + :- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -52,202 +54,210 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] +Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(35) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(38) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(37) Filter +(39) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(38) Project +(40) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(39) Exchange +(41) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(41) Project +(43) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(42) HashAggregate +(44) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(43) Exchange +(45) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) HashAggregate +(46) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(45) Exchange +(47) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(48) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(47) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 796c40d94d57..10a7818f6c4d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (51) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (50) - +- Exchange (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin LeftOuter BuildRight (42) - :- Exchange (37) - : +- Scan parquet (36) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftOuter BuildRight (43) + :- Exchange (38) + : +- Scan parquet (37) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -84,200 +85,204 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) ProjectExecTransformer +(9) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] +Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] + +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(17) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(19) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(23) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(27) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(29) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(36) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) HashAggregate +(46) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(46) HashAggregate +(47) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(47) Exchange +(48) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(49) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(49) Exchange +(50) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Sort +(51) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(51) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 180411f0b335..4a4159446f40 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ ProjectExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (32) - +- HashAggregate (31) - +- Project (30) - +- ShuffledHashJoin Inner BuildRight (29) - :- Exchange (25) - : +- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- Exchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- HashAggregate (33) + +- Project (32) + +- ShuffledHashJoin Inner BuildRight (31) + :- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -40,150 +42,158 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(23) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(24) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(26) Scan parquet +(28) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) Exchange +(30) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) ShuffledHashJoin +(31) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(32) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(33) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(33) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index 84df2af7dd53..90558947bb35 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (40) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- AQEShuffleRead (24) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner (19) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FlushableHashAggregateExecTransformer (10) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner (21) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (39) - +- Exchange (38) - +- Project (37) - +- ShuffledHashJoin Inner BuildLeft (36) - :- Exchange (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Filter (35) - +- HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- Filter (30) - +- Scan parquet (29) + Sort (41) + +- Exchange (40) + +- Project (39) + +- ShuffledHashJoin Inner BuildLeft (38) + :- Exchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -47,330 +49,343 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(11) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(13) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(19) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(24) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(25) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(27) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(28) Exchange +(30) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(31) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(32) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(33) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(35) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(36) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(37) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(38) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(40) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (64) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (55) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ ProjectExecTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ Scan parquet (41) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (63) - +- HashAggregate (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(41) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(42) ProjectExecTransformer +(44) NoopFilter +Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(43) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(51) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(53) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(54) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(55) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(56) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(57) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(58) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(60) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(63) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 456656af6264..98c7da22a549 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,62 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (62) +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ SortExecTransformer (38) - +- ^ InputIteratorTransformer (37) - +- ^ InputAdapter (36) - +- ^ ShuffleQueryStage (35), Statistics(X) - +- ColumnarExchange (34) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FlushableHashAggregateExecTransformer (25) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ^ InputAdapter (38) + +- ^ ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ^ InputAdapter (32) + +- ^ ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (61) - +- Exchange (60) - +- HashAggregate (59) - +- Exchange (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- ShuffledHashJoin Inner BuildRight (52) - :- Exchange (48) - : +- BroadcastHashJoin LeftAnti BuildRight (47) - : :- Filter (42) - : : +- Scan parquet (41) - : +- BroadcastExchange (46) - : +- Project (45) - : +- Filter (44) - : +- Scan parquet (43) - +- Exchange (51) - +- Filter (50) - +- Scan parquet (49) + Sort (63) + +- Exchange (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -66,277 +68,285 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X, ps_suppkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(28) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(30) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(32) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(33) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(34) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(36) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(37) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(38) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(39) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(40) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(43) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(44) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(45) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(46) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(47) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(48) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Scan parquet +(51) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(50) Filter +(52) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(51) Exchange +(53) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(53) Project +(55) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(54) HashAggregate +(56) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(55) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(58) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(58) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(60) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(63) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(62) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index 276e3e66628e..97c56c7dae8d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ ProjectExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ ShuffledHashJoinExecTransformer Inner (28) - :- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ FilterExecTransformer (27) - +- ^ ProjectExecTransformer (26) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ Scan parquet (17) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ^ InputAdapter (26) + +- ^ ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (53) - +- HashAggregate (52) - +- Project (51) - +- ShuffledHashJoin Inner BuildRight (50) - :- Project (43) - : +- ShuffledHashJoin Inner BuildRight (42) - : :- Exchange (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Exchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Filter (49) - +- HashAggregate (48) - +- Exchange (47) - +- HashAggregate (46) - +- Filter (45) - +- Scan parquet (44) + HashAggregate (56) + +- HashAggregate (55) + +- Project (54) + +- ShuffledHashJoin Inner BuildRight (53) + :- Project (46) + : +- ShuffledHashJoin Inner BuildRight (45) + : :- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Exchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -60,254 +63,266 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X] + +(3) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [p_partkey#X, p_brand#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_container#X] + +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) NoopFilter +Input [2]: [l_partkey#X, l_quantity#X] +Arguments: [l_partkey#X, l_quantity#X] + +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(19) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(20) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(21) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(27) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(30) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(37) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(40) Project +(43) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(41) Exchange +(44) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(43) Project +(46) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(45) Filter +(48) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(46) HashAggregate +(49) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(47) Exchange +(50) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) HashAggregate +(51) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(49) Filter +(52) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(50) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(51) Project +(54) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) HashAggregate +(55) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(53) HashAggregate +(56) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(54) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 2bda86546324..f0537d1fcf07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,93 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- TakeOrderedAndProjectExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ ProjectExecTransformer (56) - +- ^ ShuffledHashJoinExecTransformer Inner (55) - :- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ColumnarExchange (36) - : +- ^ ProjectExecTransformer (34) - : +- ^ ShuffledHashJoinExecTransformer Inner (33) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ ShuffleQueryStage (30), Statistics(X) - : +- ColumnarExchange (29) - : +- ^ ProjectExecTransformer (27) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) - : :- ^ InputIteratorTransformer (14) - : : +- ^ InputAdapter (13) - : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : +- ColumnarExchange (11) - : : +- ^ ProjectExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ ProjectExecTransformer (25) - : +- ^ FilterExecTransformer (24) - : +- ^ RegularHashAggregateExecTransformer (23) - : +- ^ InputIteratorTransformer (22) - : +- ^ InputAdapter (21) - : +- ^ ShuffleQueryStage (20), Statistics(X) - : +- ColumnarExchange (19) - : +- ^ ProjectExecTransformer (17) - : +- ^ FlushableHashAggregateExecTransformer (16) - : +- ^ Scan parquet (15) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) - :- ^ InputIteratorTransformer (46) - : +- ^ InputAdapter (45) - : +- ^ ShuffleQueryStage (44), Statistics(X) - : +- ColumnarExchange (43) - : +- ^ ProjectExecTransformer (41) - : +- ^ Scan parquet (40) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (51) - +- ^ InputIteratorTransformer (50) - +- ^ InputAdapter (49) - +- ^ ShuffleQueryStage (48), Statistics(X) - +- ReusedExchange (47) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- HashAggregate (91) - +- Project (90) - +- ShuffledHashJoin Inner BuildRight (89) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildLeft (76) - : :- Exchange (64) - : : +- Filter (63) - : : +- Scan parquet (62) - : +- Exchange (75) - : +- ShuffledHashJoin LeftSemi BuildRight (74) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Project (73) - : +- Filter (72) - : +- HashAggregate (71) - : +- Exchange (70) - : +- HashAggregate (69) - : +- Scan parquet (68) - +- ShuffledHashJoin LeftSemi BuildRight (88) - :- Exchange (81) - : +- Filter (80) - : +- Scan parquet (79) - +- Project (87) - +- Filter (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Scan parquet (82) + TakeOrderedAndProject (96) + +- HashAggregate (95) + +- HashAggregate (94) + +- Project (93) + +- ShuffledHashJoin Inner BuildRight (92) + :- Exchange (81) + : +- Project (80) + : +- ShuffledHashJoin Inner BuildLeft (79) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (78) + : +- ShuffledHashJoin LeftSemi BuildRight (77) + : :- Exchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- Project (76) + : +- Filter (75) + : +- HashAggregate (74) + : +- Exchange (73) + : +- HashAggregate (72) + : +- Scan parquet (71) + +- ShuffledHashJoin LeftSemi BuildRight (91) + :- Exchange (84) + : +- Filter (83) + : +- Scan parquet (82) + +- Project (90) + +- Filter (89) + +- HashAggregate (88) + +- Exchange (87) + +- HashAggregate (86) + +- Scan parquet (85) (1) Scan parquet @@ -97,428 +100,440 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X, c_name#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(19) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(20) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(21) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(23) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(24) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(25) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(26) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(29) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(31) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(32) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(34) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(36) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(38) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(39) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(40) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(41) ProjectExecTransformer +(43) NoopFilter +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X, l_quantity#X] + +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(42) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(43) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(45) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(46) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(47) ReusedExchange [Reuses operator id: 19] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(48) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(49) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(50) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(52) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(54) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(55) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(57) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(58) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(59) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(60) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(61) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(62) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(63) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(64) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(68) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(69) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(67) Exchange +(70) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(69) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(70) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(72) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(73) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(74) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(75) Exchange +(78) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(79) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(77) Project +(80) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(78) Exchange +(81) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(81) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(83) HashAggregate +(86) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(84) Exchange +(87) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(88) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(86) Filter +(89) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(87) Project +(90) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(88) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(89) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(90) Project +(93) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(91) HashAggregate +(94) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(92) HashAggregate +(95) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(93) TakeOrderedAndProject +(96) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(94) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index cff613274b00..7d03f6eca963 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (32) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ RegularHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer Inner (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (31) - +- HashAggregate (30) - +- Project (29) - +- ShuffledHashJoin Inner BuildRight (28) - :- Exchange (24) - : +- Project (23) - : +- Filter (22) - : +- Scan parquet (21) - +- Exchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- HashAggregate (32) + +- Project (31) + +- ShuffledHashJoin Inner BuildRight (30) + :- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Exchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -39,146 +41,154 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] +Arguments: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] + +(3) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] + +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(20) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(21) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(22) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(23) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(24) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) Scan parquet +(27) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(28) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) Exchange +(29) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) ShuffledHashJoin +(30) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(31) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(32) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(32) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 541b0f0aebab..4e67d57112e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,114 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (78) - +- AQEShuffleRead (77) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ ShuffledHashJoinExecTransformer Inner (72) - :- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ ShuffleQueryStage (62), Statistics(X) - : +- ColumnarExchange (61) - : +- ^ ProjectExecTransformer (59) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : +- ColumnarExchange (25) - : : +- ^ ProjectExecTransformer (23) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : :- ^ InputIteratorTransformer (14) - : : : +- ^ InputAdapter (13) - : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : +- ColumnarExchange (11) - : : : +- ^ ProjectExecTransformer (9) - : : : +- ^ Scan parquet (8) - : : +- ^ InputIteratorTransformer (21) - : : +- ^ InputAdapter (20) - : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : +- ColumnarExchange (18) - : : +- ^ ProjectExecTransformer (16) - : : +- ^ Scan parquet (15) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ FilterExecTransformer (44) - : +- ^ ProjectExecTransformer (43) - : +- ^ RegularHashAggregateExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) - : :- ^ InputIteratorTransformer (35) - : : +- ^ InputAdapter (34) - : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ Scan parquet (29) - : +- ^ InputIteratorTransformer (39) - : +- ^ InputAdapter (38) - : +- ^ ShuffleQueryStage (37), Statistics(X) - : +- ReusedExchange (36) - +- ^ InputIteratorTransformer (71) - +- ^ InputAdapter (70) - +- ^ ShuffleQueryStage (69), Statistics(X) - +- ColumnarExchange (68) - +- ^ ProjectExecTransformer (66) - +- ^ Scan parquet (65) + VeloxColumnarToRowExec (83) + +- AQEShuffleRead (82) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (31) + : : +- ^ InputAdapter (30) + : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : :- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ^ InputAdapter (42) + : +- ^ ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (117) - +- Exchange (116) - +- Project (115) - +- ShuffledHashJoin Inner BuildRight (114) - :- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- Exchange (106) - : +- Project (105) - : +- ShuffledHashJoin Inner BuildLeft (104) - : :- Exchange (90) - : : +- ShuffledHashJoin LeftSemi BuildRight (89) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (88) - : : +- Project (87) - : : +- Filter (86) - : : +- Scan parquet (85) - : +- Exchange (103) - : +- Filter (102) - : +- HashAggregate (101) - : +- HashAggregate (100) - : +- ShuffledHashJoin LeftSemi BuildRight (99) - : :- Exchange (94) - : : +- Project (93) - : : +- Filter (92) - : : +- Scan parquet (91) - : +- Exchange (98) - : +- Project (97) - : +- Filter (96) - : +- Scan parquet (95) - +- Exchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + Sort (122) + +- Exchange (121) + +- Project (120) + +- ShuffledHashJoin Inner BuildRight (119) + :- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin LeftSemi BuildRight (112) + : :- Exchange (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- Exchange (111) + : +- Project (110) + : +- ShuffledHashJoin Inner BuildLeft (109) + : :- Exchange (95) + : : +- ShuffledHashJoin LeftSemi BuildRight (94) + : : :- Exchange (89) + : : : +- Filter (88) + : : : +- Scan parquet (87) + : : +- Exchange (93) + : : +- Project (92) + : : +- Filter (91) + : : +- Scan parquet (90) + : +- Exchange (108) + : +- Filter (107) + : +- HashAggregate (106) + : +- HashAggregate (105) + : +- ShuffledHashJoin LeftSemi BuildRight (104) + : :- Exchange (99) + : : +- Project (98) + : : +- Filter (97) + : : +- Scan parquet (96) + : +- Exchange (103) + : +- Project (102) + : +- Filter (101) + : +- Scan parquet (100) + +- Exchange (118) + +- Project (117) + +- Filter (116) + +- Scan parquet (115) (1) Scan parquet @@ -118,510 +123,530 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] + +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(16) ProjectExecTransformer +(18) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(17) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(18) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(20) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(21) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(22) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(23) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(29) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(30) ProjectExecTransformer +(33) NoopFilter +Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] +Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] + +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(31) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(32) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(34) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(35) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(36) ReusedExchange [Reuses operator id: 18] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(37) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(38) InputAdapter +(42) InputAdapter Input [1]: [p_partkey#X] -(39) InputIteratorTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -(40) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(42) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(44) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(58) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(59) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(60) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(61) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(64) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(65) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(66) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(67) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(68) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(69) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(70) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(71) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(72) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(73) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(74) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(75) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(77) AQEShuffleRead +(82) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(78) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(79) Scan parquet +(84) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(85) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(81) Exchange +(86) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(84) Exchange +(89) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(92) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) Exchange +(93) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(90) Exchange +(95) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(92) Filter +(97) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(93) Project +(98) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(97) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(100) HashAggregate +(105) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(101) HashAggregate +(106) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(102) Filter +(107) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(103) Exchange +(108) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(105) Project +(110) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(106) Exchange +(111) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(108) Project +(113) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(109) Exchange +(114) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(112) Project +(117) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(113) Exchange +(118) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(115) Project +(120) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(116) Exchange +(121) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(122) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(118) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 58be95c2bc00..658a499a30ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,108 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (113) +AdaptiveSparkPlan (118) +- == Final Plan == - VeloxColumnarToRowExec (76) - +- ^ RegularHashAggregateExecTransformer (74) - +- ^ InputIteratorTransformer (73) - +- ^ InputAdapter (72) - +- ^ ShuffleQueryStage (71), Statistics(X) - +- ColumnarExchange (70) - +- ^ ProjectExecTransformer (68) - +- ^ FlushableHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner (65) - :- ^ InputIteratorTransformer (57) - : +- ^ InputAdapter (56) - : +- ^ ShuffleQueryStage (55), Statistics(X) - : +- ColumnarExchange (54) - : +- ^ ProjectExecTransformer (52) - : +- ^ ShuffledHashJoinExecTransformer Inner (51) - : :- ^ InputIteratorTransformer (43) - : : +- ^ InputAdapter (42) - : : +- ^ ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer Inner (37) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : +- ColumnarExchange (4) - : : : +- ^ ProjectExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (36) - : : +- ^ InputAdapter (35) - : : +- ^ ShuffleQueryStage (34), Statistics(X) - : : +- ColumnarExchange (33) - : : +- ^ ProjectExecTransformer (31) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) - : : : :- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (21) - : : : +- ^ InputAdapter (20) - : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : +- ColumnarExchange (18) - : : : +- ^ ProjectExecTransformer (16) - : : : +- ^ Scan parquet (15) - : : +- ^ InputIteratorTransformer (29) - : : +- ^ InputAdapter (28) - : : +- ^ ShuffleQueryStage (27), Statistics(X) - : : +- ColumnarExchange (26) - : : +- ^ ProjectExecTransformer (24) - : : +- ^ Scan parquet (23) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ ShuffleQueryStage (48), Statistics(X) - : +- ColumnarExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ Scan parquet (44) - +- ^ InputIteratorTransformer (64) - +- ^ InputAdapter (63) - +- ^ ShuffleQueryStage (62), Statistics(X) - +- ColumnarExchange (61) - +- ^ ProjectExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (112) - +- HashAggregate (111) - +- Exchange (110) - +- HashAggregate (109) - +- Project (108) - +- ShuffledHashJoin Inner BuildRight (107) - :- Exchange (102) - : +- Project (101) - : +- ShuffledHashJoin Inner BuildRight (100) - : :- Exchange (95) - : : +- Project (94) - : : +- ShuffledHashJoin Inner BuildLeft (93) - : : :- Exchange (79) - : : : +- Filter (78) - : : : +- Scan parquet (77) - : : +- Exchange (92) - : : +- ShuffledHashJoin LeftAnti BuildRight (91) - : : :- ShuffledHashJoin LeftSemi BuildRight (86) - : : : :- Exchange (83) - : : : : +- Project (82) - : : : : +- Filter (81) - : : : : +- Scan parquet (80) - : : : +- Exchange (85) - : : : +- Scan parquet (84) - : : +- Exchange (90) - : : +- Project (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (99) - : +- Project (98) - : +- Filter (97) - : +- Scan parquet (96) - +- Exchange (106) - +- Project (105) - +- Filter (104) - +- Scan parquet (103) + TakeOrderedAndProject (117) + +- HashAggregate (116) + +- Exchange (115) + +- HashAggregate (114) + +- Project (113) + +- ShuffledHashJoin Inner BuildRight (112) + :- Exchange (107) + : +- Project (106) + : +- ShuffledHashJoin Inner BuildRight (105) + : :- Exchange (100) + : : +- Project (99) + : : +- ShuffledHashJoin Inner BuildLeft (98) + : : :- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Exchange (97) + : : +- ShuffledHashJoin LeftAnti BuildRight (96) + : : :- ShuffledHashJoin LeftSemi BuildRight (91) + : : : :- Exchange (88) + : : : : +- Project (87) + : : : : +- Filter (86) + : : : : +- Scan parquet (85) + : : : +- Exchange (90) + : : : +- Scan parquet (89) + : : +- Exchange (95) + : : +- Project (94) + : : +- Filter (93) + : : +- Scan parquet (92) + : +- Exchange (104) + : +- Project (103) + : +- Filter (102) + : +- Scan parquet (101) + +- Exchange (111) + +- Project (110) + +- Filter (109) + +- Scan parquet (108) (1) Scan parquet @@ -112,492 +117,512 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_name#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(15) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(22) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(23) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(24) ProjectExecTransformer +(26) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] + +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(25) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(26) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(28) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(29) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(30) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(31) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(33) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(34) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(35) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(36) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(37) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(38) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(40) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(42) InputAdapter +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(43) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(44) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(45) ProjectExecTransformer +(48) NoopFilter +Input [2]: [o_orderkey#X, o_orderstatus#X] +Arguments: [o_orderkey#X, o_orderstatus#X] + +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(46) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(47) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(48) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(49) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(50) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(51) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(52) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(53) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(54) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(56) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(57) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(58) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(59) ProjectExecTransformer +(63) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(61) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(62) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(63) InputAdapter +(68) InputAdapter Input [1]: [n_nationkey#X] -(64) InputIteratorTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -(65) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(67) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(68) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(69) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(70) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(71) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(72) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(73) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(74) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(75) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(76) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(77) Scan parquet +(82) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(78) Filter +(83) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(79) Exchange +(84) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) Scan parquet +(85) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(86) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(82) Project +(87) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(83) Exchange +(88) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(89) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(87) Scan parquet +(92) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(88) Filter +(93) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(89) Project +(94) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(94) Project +(99) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(95) Exchange +(100) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(101) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(97) Filter +(102) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(98) Project +(103) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(99) Exchange +(104) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(101) Project +(106) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(102) Exchange +(107) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(104) Filter +(109) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(105) Project +(110) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(106) Exchange +(111) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(108) Project +(113) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(109) HashAggregate +(114) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(110) Exchange +(115) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) HashAggregate +(116) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(112) TakeOrderedAndProject +(117) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(113) AdaptiveSparkPlan +(118) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 5a5cb9c96758..e94b7b01715f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,46 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ^ InputAdapter (29) + +- ^ ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- ShuffledHashJoin LeftAnti BuildRight (38) - :- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (37) - +- Scan parquet (36) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- ShuffledHashJoin LeftAnti BuildRight (39) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Exchange (38) + +- Scan parquet (37) (1) Scan parquet @@ -50,302 +51,311 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X, c_phone#X, c_acctbal#X] + +(3) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(9) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(13) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(14) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(16) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(17) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(20) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(32) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(35) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Scan parquet +(37) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(37) Exchange +(38) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(39) Project +(40) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(40) HashAggregate +(41) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(43) Exchange +(44) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(45) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (65) +- == Final Plan == - VeloxColumnarToRowExec (56) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ColumnarExchange (50) - +- ^ FlushableHashAggregateExecTransformer (48) - +- ^ ProjectExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == - HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(46) Scan parquet +(47) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(47) ProjectExecTransformer +(48) NoopFilter +Input [2]: [c_phone#X, c_acctbal#X] +Arguments: [c_phone#X, c_acctbal#X] + +(49) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(48) FlushableHashAggregateExecTransformer +(50) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(49) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(50) ColumnarExchange +(52) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(52) InputAdapter +(54) InputAdapter Input [2]: [sum#X, count#X] -(53) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [sum#X, count#X] -(54) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(56) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(57) Scan parquet +(59) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(58) Filter +(60) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(59) Project +(61) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(60) HashAggregate +(62) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(61) Exchange +(63) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(63) AdaptiveSparkPlan +(65) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index 25e019dc37ba..16615ac0598f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,57 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- TakeOrderedAndProjectExecTransformer (35) - +- ^ ProjectExecTransformer (33) - +- ^ RegularHashAggregateExecTransformer (32) - +- ^ RegularHashAggregateExecTransformer (31) - +- ^ ProjectExecTransformer (30) - +- ^ ShuffledHashJoinExecTransformer Inner (29) - :- ^ InputIteratorTransformer (21) - : +- ^ InputAdapter (20) - : +- ^ ShuffleQueryStage (19), Statistics(X) - : +- ColumnarExchange (18) - : +- ^ ProjectExecTransformer (16) - : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : +- ColumnarExchange (4) - : : +- ^ ProjectExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (14) - : +- ^ InputAdapter (13) - : +- ^ ShuffleQueryStage (12), Statistics(X) - : +- ColumnarExchange (11) - : +- ^ ProjectExecTransformer (9) - : +- ^ Scan parquet (8) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ Scan parquet (22) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (55) - +- HashAggregate (54) - +- HashAggregate (53) - +- Project (52) - +- ShuffledHashJoin Inner BuildRight (51) - :- Exchange (46) - : +- Project (45) - : +- ShuffledHashJoin Inner BuildLeft (44) - : :- Exchange (40) - : : +- Project (39) - : : +- Filter (38) - : : +- Scan parquet (37) - : +- Exchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Exchange (50) - +- Project (49) - +- Filter (48) - +- Scan parquet (47) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -61,248 +64,260 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_mktsegment#X] +Arguments: [c_custkey#X, c_mktsegment#X] + +(3) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] + +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(32) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(33) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(37) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(38) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(39) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(40) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(43) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(45) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(46) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(49) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(52) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(53) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(54) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(55) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index f57e74a85df5..1d3f8903f89f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,49 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ RegularHashAggregateExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ ProjectExecTransformer (18) - +- ^ FlushableHashAggregateExecTransformer (17) - +- ^ ProjectExecTransformer (16) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (14) - +- ^ InputAdapter (13) - +- ^ ShuffleQueryStage (12), Statistics(X) - +- ColumnarExchange (11) - +- ^ ProjectExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + :- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- HashAggregate (45) - +- Exchange (44) - +- HashAggregate (43) - +- Project (42) - +- ShuffledHashJoin LeftSemi BuildRight (41) - :- Exchange (36) - : +- Project (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Exchange (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftSemi BuildRight (43) + :- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -53,206 +55,214 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] +Arguments: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] + +(3) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] +Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] + +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(17) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(20) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(24) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(26) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(28) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(29) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(30) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(32) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(35) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(36) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(39) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(38) Filter +(40) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(39) Project +(41) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(40) Exchange +(42) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(42) Project +(44) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(43) HashAggregate +(45) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(44) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(46) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 9de96b917fad..12c4c6add240 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,121 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (134) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (120) - +- Project (119) - +- Filter (118) - +- Scan parquet (117) + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- ShuffledHashJoin Inner BuildRight (127) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (126) + +- Project (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -125,550 +131,574 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(22) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] + +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [1]: [r_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(91) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(98) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(93) Filter +(99) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(94) Project +(100) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(95) Exchange +(101) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(97) Project +(103) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(98) Exchange +(104) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(101) Exchange +(107) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(103) Project +(109) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(110) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(109) Project +(115) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(116) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(113) Exchange +(119) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(115) Project +(121) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(116) Exchange +(122) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(119) Project +(125) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(120) Exchange +(126) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(127) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(122) Project +(128) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(123) HashAggregate +(129) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(124) Exchange +(130) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(131) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(126) Exchange +(132) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(133) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(128) AdaptiveSparkPlan +(134) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 733adf0d0b4a..9333e2a8ad59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (18) +AdaptiveSparkPlan (19) +- == Final Plan == - VeloxColumnarToRowExec (11) - +- ^ RegularHashAggregateExecTransformer (9) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ ShuffleQueryStage (6), Statistics(X) - +- ColumnarExchange (5) - +- ^ FlushableHashAggregateExecTransformer (3) - +- ^ ProjectExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (17) - +- Exchange (16) - +- HashAggregate (15) - +- Project (14) - +- Filter (13) - +- Scan parquet (12) + HashAggregate (18) + +- Exchange (17) + +- HashAggregate (16) + +- Project (15) + +- Filter (14) + +- Scan parquet (13) (1) Scan parquet @@ -26,82 +27,86 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(3) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(3) FlushableHashAggregateExecTransformer +(4) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(4) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(5) ColumnarExchange +(6) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [sum#X, isEmpty#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(9) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(10) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(11) VeloxColumnarToRowExec +(12) VeloxColumnarToRowExec Input [1]: [revenue#X] -(12) Scan parquet +(13) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(13) Filter +(14) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(14) Project +(15) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) HashAggregate +(16) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(16) Exchange +(17) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(17) HashAggregate +(18) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(18) AdaptiveSparkPlan +(19) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 50c9b4a231f1..03305d572de7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,117 +1,122 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (85) - +- ^ SortExecTransformer (83) - +- ^ InputIteratorTransformer (82) - +- ^ InputAdapter (81) - +- ^ ShuffleQueryStage (80), Statistics(X) - +- ColumnarExchange (79) - +- ^ RegularHashAggregateExecTransformer (77) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FlushableHashAggregateExecTransformer (70) - +- ^ ProjectExecTransformer (69) - +- ^ ShuffledHashJoinExecTransformer Inner (68) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65), Statistics(X) - +- ReusedExchange (64) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ^ InputAdapter (86) + +- ^ ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ^ InputAdapter (80) + +- ^ ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner (73) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ^ InputAdapter (71) + +- ^ ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- HashAggregate (120) - +- Exchange (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (112) - : +- Project (111) - : +- ShuffledHashJoin Inner BuildRight (110) - : :- Exchange (106) - : : +- Project (105) - : : +- ShuffledHashJoin Inner BuildRight (104) - : : :- Exchange (100) - : : : +- Project (99) - : : : +- ShuffledHashJoin Inner BuildRight (98) - : : : :- Exchange (94) - : : : : +- Project (93) - : : : : +- ShuffledHashJoin Inner BuildLeft (92) - : : : : :- Exchange (88) - : : : : : +- Filter (87) - : : : : : +- Scan parquet (86) - : : : : +- Exchange (91) - : : : : +- Filter (90) - : : : : +- Scan parquet (89) - : : : +- Exchange (97) - : : : +- Filter (96) - : : : +- Scan parquet (95) - : : +- Exchange (103) - : : +- Filter (102) - : : +- Scan parquet (101) - : +- Exchange (109) - : +- Filter (108) - : +- Scan parquet (107) - +- Exchange (115) - +- Filter (114) - +- Scan parquet (113) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildRight (109) + : : :- Exchange (105) + : : : +- Project (104) + : : : +- ShuffledHashJoin Inner BuildRight (103) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- ShuffledHashJoin Inner BuildLeft (97) + : : : : :- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -121,526 +126,546 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(3) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X, o_custkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(64) ReusedExchange [Reuses operator id: 53] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(65) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(66) InputAdapter +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(67) InputIteratorTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(68) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(69) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(70) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(71) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(72) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(73) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(75) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(78) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(79) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(80) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(81) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(82) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(83) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(84) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(85) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(86) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(87) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(88) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(94) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(95) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) Exchange +(96) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(93) Project +(98) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(94) Exchange +(99) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(100) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(96) Filter +(101) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(97) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(99) Project +(104) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(100) Exchange +(105) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(102) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(103) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(105) Project +(110) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(106) Exchange +(111) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) Scan parquet +(112) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(108) Filter +(113) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(109) Exchange +(114) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(111) Project +(116) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(112) Exchange +(117) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(115) Exchange +(120) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(117) Project +(122) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(118) HashAggregate +(123) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(119) Exchange +(124) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) HashAggregate +(125) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(121) Exchange +(126) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(127) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index 5ed8ee8b6b4f..f0176bc5e011 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,158 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (169) +AdaptiveSparkPlan (177) +- == Final Plan == - VeloxColumnarToRowExec (117) - +- ^ SortExecTransformer (115) - +- ^ InputIteratorTransformer (114) - +- ^ InputAdapter (113) - +- ^ ShuffleQueryStage (112), Statistics(X) - +- ColumnarExchange (111) - +- ^ ProjectExecTransformer (109) - +- ^ RegularHashAggregateExecTransformer (108) - +- ^ InputIteratorTransformer (107) - +- ^ InputAdapter (106) - +- ^ ShuffleQueryStage (105), Statistics(X) - +- ColumnarExchange (104) - +- ^ ProjectExecTransformer (102) - +- ^ FlushableHashAggregateExecTransformer (101) - +- ^ ProjectExecTransformer (100) - +- ^ ShuffledHashJoinExecTransformer Inner (99) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (77) - : : +- ^ InputAdapter (76) - : : +- ^ ShuffleQueryStage (75), Statistics(X) - : : +- ColumnarExchange (74) - : : +- ^ ProjectExecTransformer (72) - : : +- ^ ShuffledHashJoinExecTransformer Inner (71) - : : :- ^ InputIteratorTransformer (63) - : : : +- ^ InputAdapter (62) - : : : +- ^ ShuffleQueryStage (61), Statistics(X) - : : : +- ColumnarExchange (60) - : : : +- ^ ProjectExecTransformer (58) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : : : :- ^ InputIteratorTransformer (49) - : : : : +- ^ InputAdapter (48) - : : : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : : : +- ColumnarExchange (46) - : : : : +- ^ ProjectExecTransformer (44) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : : : :- ^ InputIteratorTransformer (35) - : : : : : +- ^ InputAdapter (34) - : : : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : : : +- ColumnarExchange (32) - : : : : : +- ^ ProjectExecTransformer (30) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : : : :- ^ InputIteratorTransformer (21) - : : : : : : +- ^ InputAdapter (20) - : : : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : : : +- ColumnarExchange (18) - : : : : : : +- ^ ProjectExecTransformer (16) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : : : :- ^ InputIteratorTransformer (7) - : : : : : : : +- ^ InputAdapter (6) - : : : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : : : +- ColumnarExchange (4) - : : : : : : : +- ^ ProjectExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (14) - : : : : : : +- ^ InputAdapter (13) - : : : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : : : +- ColumnarExchange (11) - : : : : : : +- ^ ProjectExecTransformer (9) - : : : : : : +- ^ Scan parquet (8) - : : : : : +- ^ InputIteratorTransformer (28) - : : : : : +- ^ InputAdapter (27) - : : : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : : : +- ColumnarExchange (25) - : : : : : +- ^ ProjectExecTransformer (23) - : : : : : +- ^ Scan parquet (22) - : : : : +- ^ InputIteratorTransformer (42) - : : : : +- ^ InputAdapter (41) - : : : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : : : +- ColumnarExchange (39) - : : : : +- ^ ProjectExecTransformer (37) - : : : : +- ^ Scan parquet (36) - : : : +- ^ InputIteratorTransformer (56) - : : : +- ^ InputAdapter (55) - : : : +- ^ ShuffleQueryStage (54), Statistics(X) - : : : +- ColumnarExchange (53) - : : : +- ^ ProjectExecTransformer (51) - : : : +- ^ Scan parquet (50) - : : +- ^ InputIteratorTransformer (70) - : : +- ^ InputAdapter (69) - : : +- ^ ShuffleQueryStage (68), Statistics(X) - : : +- ColumnarExchange (67) - : : +- ^ ProjectExecTransformer (65) - : : +- ^ Scan parquet (64) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ColumnarExchange (81) - : +- ^ ProjectExecTransformer (79) - : +- ^ Scan parquet (78) - +- ^ InputIteratorTransformer (98) - +- ^ InputAdapter (97) - +- ^ ShuffleQueryStage (96), Statistics(X) - +- ColumnarExchange (95) - +- ^ ProjectExecTransformer (93) - +- ^ Scan parquet (92) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ^ InputAdapter (121) + +- ^ ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ^ InputAdapter (114) + +- ^ ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner (107) + :- ^ InputIteratorTransformer (98) + : +- ^ InputAdapter (97) + : +- ^ ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : :- ^ InputIteratorTransformer (83) + : : +- ^ InputAdapter (82) + : : +- ^ ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ^ InputAdapter (67) + : : : +- ^ ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ^ InputAdapter (52) + : : : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ^ InputAdapter (37) + : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ^ InputAdapter (22) + : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ^ InputAdapter (7) + : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ^ InputAdapter (15) + : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ^ InputAdapter (30) + : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ^ InputAdapter (45) + : : : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ^ InputAdapter (60) + : : : +- ^ ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ^ InputAdapter (105) + +- ^ ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (168) - +- Exchange (167) - +- HashAggregate (166) - +- Exchange (165) - +- HashAggregate (164) - +- Project (163) - +- ShuffledHashJoin Inner BuildRight (162) - :- Exchange (157) - : +- Project (156) - : +- ShuffledHashJoin Inner BuildRight (155) - : :- Exchange (151) - : : +- Project (150) - : : +- ShuffledHashJoin Inner BuildRight (149) - : : :- Exchange (145) - : : : +- Project (144) - : : : +- ShuffledHashJoin Inner BuildRight (143) - : : : :- Exchange (139) - : : : : +- Project (138) - : : : : +- ShuffledHashJoin Inner BuildRight (137) - : : : : :- Exchange (133) - : : : : : +- Project (132) - : : : : : +- ShuffledHashJoin Inner BuildRight (131) - : : : : : :- Exchange (127) - : : : : : : +- Project (126) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) - : : : : : : :- Exchange (121) - : : : : : : : +- Project (120) - : : : : : : : +- Filter (119) - : : : : : : : +- Scan parquet (118) - : : : : : : +- Exchange (124) - : : : : : : +- Filter (123) - : : : : : : +- Scan parquet (122) - : : : : : +- Exchange (130) - : : : : : +- Filter (129) - : : : : : +- Scan parquet (128) - : : : : +- Exchange (136) - : : : : +- Filter (135) - : : : : +- Scan parquet (134) - : : : +- Exchange (142) - : : : +- Filter (141) - : : : +- Scan parquet (140) - : : +- Exchange (148) - : : +- Filter (147) - : : +- Scan parquet (146) - : +- Exchange (154) - : +- Filter (153) - : +- Scan parquet (152) - +- Exchange (161) - +- Project (160) - +- Filter (159) - +- Scan parquet (158) + Sort (176) + +- Exchange (175) + +- HashAggregate (174) + +- Exchange (173) + +- HashAggregate (172) + +- Project (171) + +- ShuffledHashJoin Inner BuildRight (170) + :- Exchange (165) + : +- Project (164) + : +- ShuffledHashJoin Inner BuildRight (163) + : :- Exchange (159) + : : +- Project (158) + : : +- ShuffledHashJoin Inner BuildRight (157) + : : :- Exchange (153) + : : : +- Project (152) + : : : +- ShuffledHashJoin Inner BuildRight (151) + : : : :- Exchange (147) + : : : : +- Project (146) + : : : : +- ShuffledHashJoin Inner BuildRight (145) + : : : : :- Exchange (141) + : : : : : +- Project (140) + : : : : : +- ShuffledHashJoin Inner BuildRight (139) + : : : : : :- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) + : : : : : : :- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Exchange (132) + : : : : : : +- Filter (131) + : : : : : : +- Scan parquet (130) + : : : : : +- Exchange (138) + : : : : : +- Filter (137) + : : : : : +- Scan parquet (136) + : : : : +- Exchange (144) + : : : : +- Filter (143) + : : : : +- Scan parquet (142) + : : : +- Exchange (150) + : : : +- Filter (149) + : : : +- Scan parquet (148) + : : +- Exchange (156) + : : +- Filter (155) + : : +- Scan parquet (154) + : +- Exchange (162) + : +- Filter (161) + : +- Scan parquet (160) + +- Exchange (169) + +- Project (168) + +- Filter (167) + +- Scan parquet (166) (1) Scan parquet @@ -162,726 +170,758 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X, p_type#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] + +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X, c_nationkey#X] + +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X, n_regionkey#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(73) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(74) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(76) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(77) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) ProjectExecTransformer +(85) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(80) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(81) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(92) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(93) ProjectExecTransformer +(100) NoopFilter +Input [2]: [r_regionkey#X, r_name#X] +Arguments: [r_regionkey#X, r_name#X] + +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(94) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(95) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(97) InputAdapter +(105) InputAdapter Input [1]: [r_regionkey#X] -(98) InputIteratorTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -(99) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(101) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(102) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(103) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(104) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(105) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(106) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(107) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(108) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(109) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(110) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(111) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(112) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(113) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(114) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(115) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(116) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(117) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(118) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(120) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(121) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Scan parquet +(130) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(123) Filter +(131) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(124) Exchange +(132) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(133) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(126) Project +(134) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(127) Exchange +(135) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(129) Filter +(137) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(130) Exchange +(138) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(132) Project +(140) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(133) Exchange +(141) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) Scan parquet +(142) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(135) Filter +(143) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(136) Exchange +(144) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) ShuffledHashJoin +(145) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(138) Project +(146) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(139) Exchange +(147) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Scan parquet +(148) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(141) Filter +(149) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(142) Exchange +(150) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) ShuffledHashJoin +(151) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(144) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(145) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(154) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(147) Filter +(155) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(148) Exchange +(156) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(157) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(150) Project +(158) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(151) Exchange +(159) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(160) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(153) Filter +(161) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(154) Exchange +(162) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(163) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(156) Project +(164) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(157) Exchange +(165) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(166) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(159) Filter +(167) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(160) Project +(168) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(161) Exchange +(169) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(162) ShuffledHashJoin +(170) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(163) Project +(171) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(164) HashAggregate +(172) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(165) Exchange +(173) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) HashAggregate +(174) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(167) Exchange +(175) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(176) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(169) AdaptiveSparkPlan +(177) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 0d3c59505d07..3f5479af2b14 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,120 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (127) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ RegularHashAggregateExecTransformer (80) - +- ^ InputIteratorTransformer (79) - +- ^ InputAdapter (78) - +- ^ ShuffleQueryStage (77), Statistics(X) - +- ColumnarExchange (76) - +- ^ ProjectExecTransformer (74) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ ShuffledHashJoinExecTransformer Inner (71) - :- ^ InputIteratorTransformer (63) - : +- ^ InputAdapter (62) - : +- ^ ShuffleQueryStage (61), Statistics(X) - : +- ColumnarExchange (60) - : +- ^ ProjectExecTransformer (58) - : +- ^ ShuffledHashJoinExecTransformer Inner (57) - : :- ^ InputIteratorTransformer (49) - : : +- ^ InputAdapter (48) - : : +- ^ ShuffleQueryStage (47), Statistics(X) - : : +- ColumnarExchange (46) - : : +- ^ ProjectExecTransformer (44) - : : +- ^ ShuffledHashJoinExecTransformer Inner (43) - : : :- ^ InputIteratorTransformer (35) - : : : +- ^ InputAdapter (34) - : : : +- ^ ShuffleQueryStage (33), Statistics(X) - : : : +- ColumnarExchange (32) - : : : +- ^ ProjectExecTransformer (30) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) - : : : :- ^ InputIteratorTransformer (21) - : : : : +- ^ InputAdapter (20) - : : : : +- ^ ShuffleQueryStage (19), Statistics(X) - : : : : +- ColumnarExchange (18) - : : : : +- ^ ProjectExecTransformer (16) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) - : : : : : +- ColumnarExchange (4) - : : : : : +- ^ ProjectExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (14) - : : : : +- ^ InputAdapter (13) - : : : : +- ^ ShuffleQueryStage (12), Statistics(X) - : : : : +- ColumnarExchange (11) - : : : : +- ^ ProjectExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (28) - : : : +- ^ InputAdapter (27) - : : : +- ^ ShuffleQueryStage (26), Statistics(X) - : : : +- ColumnarExchange (25) - : : : +- ^ ProjectExecTransformer (23) - : : : +- ^ Scan parquet (22) - : : +- ^ InputIteratorTransformer (42) - : : +- ^ InputAdapter (41) - : : +- ^ ShuffleQueryStage (40), Statistics(X) - : : +- ColumnarExchange (39) - : : +- ^ ProjectExecTransformer (37) - : : +- ^ Scan parquet (36) - : +- ^ InputIteratorTransformer (56) - : +- ^ InputAdapter (55) - : +- ^ ShuffleQueryStage (54), Statistics(X) - : +- ColumnarExchange (53) - : +- ^ ProjectExecTransformer (51) - : +- ^ Scan parquet (50) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ ShuffleQueryStage (68), Statistics(X) - +- ColumnarExchange (67) - +- ^ ProjectExecTransformer (65) - +- ^ Scan parquet (64) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner (77) + :- ^ InputIteratorTransformer (68) + : +- ^ InputAdapter (67) + : +- ^ ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : :- ^ InputIteratorTransformer (53) + : : +- ^ InputAdapter (52) + : : +- ^ ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ^ InputAdapter (37) + : : : +- ^ ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ^ InputAdapter (22) + : : : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ^ InputAdapter (7) + : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ^ InputAdapter (30) + : : : +- ^ ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (126) - +- Exchange (125) - +- HashAggregate (124) - +- Exchange (123) - +- HashAggregate (122) - +- Project (121) - +- ShuffledHashJoin Inner BuildRight (120) - :- Exchange (116) - : +- Project (115) - : +- ShuffledHashJoin Inner BuildRight (114) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildRight (108) - : : :- Exchange (104) - : : : +- Project (103) - : : : +- ShuffledHashJoin Inner BuildRight (102) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- ShuffledHashJoin Inner BuildLeft (96) - : : : : :- Exchange (92) - : : : : : +- Project (91) - : : : : : +- Filter (90) - : : : : : +- Scan parquet (89) - : : : : +- Exchange (95) - : : : : +- Filter (94) - : : : : +- Scan parquet (93) - : : : +- Exchange (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Exchange (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Exchange (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (119) - +- Filter (118) - +- Scan parquet (117) + Sort (132) + +- Exchange (131) + +- HashAggregate (130) + +- Exchange (129) + +- HashAggregate (128) + +- Project (127) + +- ShuffledHashJoin Inner BuildRight (126) + :- Exchange (122) + : +- Project (121) + : +- ShuffledHashJoin Inner BuildRight (120) + : :- Exchange (116) + : : +- Project (115) + : : +- ShuffledHashJoin Inner BuildRight (114) + : : :- Exchange (110) + : : : +- Project (109) + : : : +- ShuffledHashJoin Inner BuildRight (108) + : : : :- Exchange (104) + : : : : +- Project (103) + : : : : +- ShuffledHashJoin Inner BuildLeft (102) + : : : : :- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Exchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) (1) Scan parquet @@ -124,546 +130,570 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) ProjectExecTransformer +(2) NoopFilter +Input [2]: [p_partkey#X, p_name#X] +Arguments: [p_partkey#X, p_name#X] + +(3) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(3) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(4) ColumnarExchange +(5) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [p_partkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [p_partkey#X] -(8) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) ProjectExecTransformer +(10) NoopFilter +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] + +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(10) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(11) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(12) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(13) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(14) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(15) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(16) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(18) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(19) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(20) InputAdapter +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) InputIteratorTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) ProjectExecTransformer +(25) NoopFilter +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X, s_nationkey#X] + +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(24) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(32) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(34) InputAdapter +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(35) InputIteratorTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(36) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(37) ProjectExecTransformer +(40) NoopFilter +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] + +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(38) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(39) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(41) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(43) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(46) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(48) InputAdapter +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(49) InputIteratorTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(50) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(51) ProjectExecTransformer +(55) NoopFilter +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X, o_orderdate#X] + +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(52) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(53) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(55) InputAdapter +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(56) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(58) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(59) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(60) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(61) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(62) InputAdapter +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(63) InputIteratorTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(64) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) ProjectExecTransformer +(70) NoopFilter +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X, n_name#X] + +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(66) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(67) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(68) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(69) InputAdapter +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(70) InputIteratorTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(71) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(73) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(74) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(75) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(76) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(77) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(78) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(79) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(81) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(82) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(84) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(85) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(86) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(89) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(90) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(91) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(92) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(99) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(94) Filter +(100) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(95) Exchange +(101) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(97) Project +(103) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(98) Exchange +(104) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(100) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(101) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(103) Project +(109) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(104) Exchange +(110) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(111) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(106) Filter +(112) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(107) Exchange +(113) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(109) Project +(115) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(110) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(113) Exchange +(119) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(115) Project +(121) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(116) Exchange +(122) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(118) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(119) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(127) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(122) HashAggregate +(128) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(123) Exchange +(129) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) HashAggregate +(130) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(125) Exchange +(131) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(132) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(127) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 17f27a407e52..792ab407836c 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -329,7 +329,7 @@ class VeloxTPCHV1RasSuite extends VeloxTPCHSuite { super.sparkConf .set("spark.sql.sources.useV1SourceList", "parquet") .set("spark.sql.autoBroadcastJoinThreshold", "-1") - .set("spark.gluten.sql.ras.enabled", "true") + .set("spark.gluten.ras.enabled", "true") } } @@ -340,7 +340,7 @@ class VeloxTPCHV1BhjRasSuite extends VeloxTPCHSuite { super.sparkConf .set("spark.sql.sources.useV1SourceList", "parquet") .set("spark.sql.autoBroadcastJoinThreshold", "30M") - .set("spark.gluten.sql.ras.enabled", "true") + .set("spark.gluten.ras.enabled", "true") } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala index ae2cea0ba03d..1806eacfc677 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala @@ -16,10 +16,12 @@ */ package org.apache.gluten.planner +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.columnar.transition.ConventionReq +import org.apache.gluten.planner.cost.GlutenCostModel import org.apache.gluten.planner.property.Conv +import org.apache.gluten.ras.{Cost, CostModel, Ras} import org.apache.gluten.ras.Best.BestNotFoundException -import org.apache.gluten.ras.Ras import org.apache.gluten.ras.RasSuiteBase._ import org.apache.gluten.ras.path.RasPath import org.apache.gluten.ras.property.PropertySet @@ -117,15 +119,46 @@ class VeloxRasSuite extends SharedSparkSession { planner2.plan() } } + + test("User cost model") { + withSQLConf(GlutenConfig.RAS_COST_MODEL.key -> classOf[UserCostModel1].getName) { + val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) + val planner = newRas(List(RowUnaryToColumnarUnary)).newPlanner(in) + val out = planner.plan() + assert(out == ColumnarUnary(RowToColumnarExec(RowLeaf(TRIVIAL_SCHEMA)))) + } + withSQLConf(GlutenConfig.RAS_COST_MODEL.key -> classOf[UserCostModel2].getName) { + val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) + val planner = newRas(List(RowUnaryToColumnarUnary)).newPlanner(in) + val out = planner.plan() + assert(out == RowUnary(RowLeaf(TRIVIAL_SCHEMA))) + } + withSQLConf(GlutenConfig.RAS_COST_MODEL.key -> "user.dummy.CostModel") { + val in = RowUnary(RowLeaf(TRIVIAL_SCHEMA)) + assertThrows[ClassNotFoundException] { + newRas().newPlanner(in) + } + } + } } object VeloxRasSuite { def newRas(): Ras[SparkPlan] = { - GlutenOptimization(List()).asInstanceOf[Ras[SparkPlan]] + GlutenOptimization + .builder() + .costModel(GlutenCostModel.find()) + .addRules(List()) + .create() + .asInstanceOf[Ras[SparkPlan]] } def newRas(RasRules: Seq[RasRule[SparkPlan]]): Ras[SparkPlan] = { - GlutenOptimization(RasRules).asInstanceOf[Ras[SparkPlan]] + GlutenOptimization + .builder() + .costModel(GlutenCostModel.find()) + .addRules(RasRules) + .create() + .asInstanceOf[Ras[SparkPlan]] } val TRIVIAL_SCHEMA: Seq[AttributeReference] = List(AttributeReference("value", StringType)()) @@ -152,4 +185,32 @@ object VeloxRasSuite { override protected def withNewChildInternal(newChild: SparkPlan): ColumnarUnary = copy(child = newChild) } + + object RowUnaryToColumnarUnary extends RasRule[SparkPlan] { + override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { + case RowUnary(child) => List(ColumnarUnary(child)) + case _ => List.empty + } + override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) + } + + class UserCostModel1 extends CostModel[SparkPlan] { + private val base = GlutenCostModel.rough() + override def costOf(node: SparkPlan): Cost = node match { + case _: RowUnary => base.makeInfCost() + case other => base.costOf(other) + } + override def costComparator(): Ordering[Cost] = base.costComparator() + override def makeInfCost(): Cost = base.makeInfCost() + } + + class UserCostModel2 extends CostModel[SparkPlan] { + private val base = GlutenCostModel.rough() + override def costOf(node: SparkPlan): Cost = node match { + case _: ColumnarUnary => base.makeInfCost() + case other => base.costOf(other) + } + override def costComparator(): Ordering[Cost] = base.costComparator() + override def makeInfCost(): Cost = base.makeInfCost() + } } diff --git a/docs/Configuration.md b/docs/Configuration.md index f23ca3b66363..a148ec3aa4cb 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -22,7 +22,7 @@ You can add these configurations into spark-defaults.conf to enable or disable t | spark.plugins | To load Gluten's components by Spark's plug-in loader | org.apache.gluten.GlutenPlugin | | spark.shuffle.manager | To turn on Gluten Columnar Shuffle Plugin | org.apache.spark.shuffle.sort.ColumnarShuffleManager | | spark.gluten.enabled | Enable Gluten, default is true. Just an experimental property. Recommend to enable/disable Gluten through the setting for `spark.plugins`. | true | -| spark.gluten.sql.ras.enabled | Experimental: Enables RAS (relation algebra selector) during physical planning to generate more efficient query plan. Note, this feature is still in development and may not bring performance profits. | false | +| spark.gluten.ras.enabled | Experimental: Enables RAS (relation algebra selector) during physical planning to generate more efficient query plan. Note, this feature is still in development and may not bring performance profits. | false | | spark.gluten.sql.columnar.maxBatchSize | Number of rows to be processed in each batch. Default value is 4096. | 4096 | | spark.gluten.memory.isolation | (Experimental) Enable isolated memory mode. If true, Gluten controls the maximum off-heap memory can be used by each task to X, X = executor memory / max task slots. It's recommended to set true if Gluten serves concurrent queries within a single session, since not all memory Gluten allocated is guaranteed to be spillable. In the case, the feature should be enabled to avoid OOM. Note when true, setting spark.memory.storageFraction to a lower value is suggested since storage memory is considered non-usable by Gluten. | false | | spark.gluten.sql.columnar.scanOnly | When enabled, this config will overwrite all other operators' enabling, and only Scan and Filter pushdown will be offloaded to native. | false | diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index 3f8ee870609e..a259641f5049 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -45,35 +45,23 @@ class EnumeratedApplier(session: SparkSession) extends ColumnarRuleApplier with Logging with LogLevelUtil { + // An empirical value. + private val aqeStackTraceIndex = 16 private lazy val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel private lazy val planChangeLogger = new PlanChangeLogger[SparkPlan]() - private val adaptiveContext = AdaptiveContext(session) + private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = - withTransformRules(transformRules(outputsColumnar)).apply(plan) - - // Visible for testing. - private def withTransformRules( - transformRules: List[SparkSession => Rule[SparkPlan]]): Rule[SparkPlan] = - plan => - PhysicalPlanSelector.maybe(session, plan) { - val finalPlan = prepareFallback(plan) { - p => - val suggestedPlan = transformPlan(transformRules, p, "transform") - transformPlan(fallbackPolicies(), suggestedPlan, "fallback") match { - case FallbackNode(fallbackPlan) => - // we should use vanilla c2r rather than native c2r, - // and there should be no `GlutenPlan` any more, - // so skip the `postRules()`. - fallbackPlan - case plan => - transformPlan(postRules(), plan, "post") - } - } - transformPlan(finalRules(), finalPlan, "final") + PhysicalPlanSelector.maybe(session, plan) { + val transformed = transformPlan(transformRules(outputsColumnar), plan, "transform") + val postPlan = maybeAqe { + transformPlan(postRules(), transformed, "post") } + val finalPlan = transformPlan(finalRules(), postPlan, "final") + finalPlan + } private def transformPlan( getRules: List[SparkSession => Rule[SparkPlan]], @@ -95,13 +83,11 @@ class EnumeratedApplier(session: SparkSession) overridden }(t => logOnLevel(transformPlanLogLevel, s"${step}Transform SparkPlan took: $t ms.")) - private def prepareFallback[T](plan: SparkPlan)(f: SparkPlan => T): T = { + private def maybeAqe[T](f: => T): T = { adaptiveContext.setAdaptiveContext() - adaptiveContext.setOriginalPlan(plan) try { - f(plan) + f } finally { - adaptiveContext.resetOriginalPlan() adaptiveContext.resetAdaptiveContext() } } @@ -114,7 +100,6 @@ class EnumeratedApplier(session: SparkSession) List( (_: SparkSession) => RemoveTransitions, (spark: SparkSession) => FallbackOnANSIMode(spark), - (spark: SparkSession) => FallbackMultiCodegens(spark), (spark: SparkSession) => PlanOneRowRelation(spark), (_: SparkSession) => FallbackEmptySchemaRelation() ) ::: @@ -136,16 +121,6 @@ class EnumeratedApplier(session: SparkSession) List((_: SparkSession) => InsertTransitions(outputsColumnar)) } - /** - * Rules to add wrapper `FallbackNode`s on top of the input plan, as hints to make planner fall - * back the whole input plan to the original vanilla Spark plan. - */ - private def fallbackPolicies(): List[SparkSession => Rule[SparkPlan]] = { - List( - (_: SparkSession) => - ExpandFallbackPolicy(adaptiveContext.isAdaptiveContext(), adaptiveContext.originalPlan())) - } - /** * Rules applying to non-fallen-back Gluten plans. To do some post cleanup works on the plan to * make sure it be able to run and be compatible with Spark's execution engine. diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index c41c1ca2caa0..0b9dcc663246 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -19,6 +19,7 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.extension.columnar.{OffloadExchange, OffloadJoin, OffloadOthers} import org.apache.gluten.extension.columnar.transition.ConventionReq import org.apache.gluten.planner.GlutenOptimization +import org.apache.gluten.planner.cost.GlutenCostModel import org.apache.gluten.planner.property.Conv import org.apache.gluten.ras.property.PropertySet import org.apache.gluten.sql.shims.SparkShimLoader @@ -79,7 +80,13 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) RasOffload.from[EvalPythonExec](OffloadOthers()).toRule ) - private val optimization = GlutenOptimization(rules ++ offloadRules) + private val optimization = { + GlutenOptimization + .builder() + .costModel(GlutenCostModel.find()) + .addRules(rules ++ offloadRules) + .create() + } private val reqConvention = Conv.any diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index 46b3b7f9e088..a3b4831a622d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -16,13 +16,19 @@ */ package org.apache.gluten.extension.columnar.enumerated -import org.apache.gluten.execution.{BasicScanExecTransformer, FilterExecTransformerBase} +import org.apache.gluten.execution._ +import org.apache.gluten.metrics.{MetricsUpdater, NoopMetricsUpdater} import org.apache.gluten.ras.path.Pattern._ import org.apache.gluten.ras.path.Pattern.Matchers._ import org.apache.gluten.ras.rule.{RasRule, Shape} import org.apache.gluten.ras.rule.Shapes._ +import org.apache.gluten.substrait.SubstraitContext +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.physical.Partitioning import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.vectorized.ColumnarBatch // Removes Gluten filter operator if its no-op. Typically a Gluten filter is no-op when it // pushes all of its conditions into the child scan. @@ -35,7 +41,8 @@ object RemoveFilter extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = { val filter = node.asInstanceOf[FilterExecTransformerBase] if (filter.isNoop()) { - return List(filter.child) + val out = NoopFilter(filter.child, filter.output) + return List(out) } List.empty } @@ -46,4 +53,30 @@ object RemoveFilter extends RasRule[SparkPlan] { clazz(classOf[FilterExecTransformerBase]), leaf(clazz(classOf[BasicScanExecTransformer])) ).build()) + + // A noop filter placeholder that indicates that all conditions are pushed into scan. + // + // This operator has zero cost in cost model to avoid planner from choosing the + // original filter-scan that doesn't have all conditions pushed down to scan. + // + // We cannot simplify remove the filter to let planner choose the scan since by vanilla + // Spark's definition the filter may have different output nullability than scan. So + // we have to keep this empty filter to let the optimized tree have the identical output schema + // with the original tree. If we simply remove the filter, possible UBs might be caused. For + // example, redundant broadcast exchanges may be added by EnsureRequirements because the + // broadcast join detects that its join keys' nullabilities have been changed. Then AQE + // re-optimization could be broken by ValidateSparkPlan so that AQE could completely + // have no effect as if it's off. This case can be observed by explicitly setting a higher + // AQE logger level to make sure the validation log doesn't get suppressed, e.g., + // spark.sql.adaptive.logLevel=ERROR. + case class NoopFilter(override val child: SparkPlan, override val output: Seq[Attribute]) + extends UnaryTransformSupport { + override def metricsUpdater(): MetricsUpdater = NoopMetricsUpdater + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(newChild) + override def outputPartitioning: Partitioning = child.outputPartitioning + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + override def doTransform(context: SubstraitContext): TransformContext = + child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = child.executeColumnar() + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index 2b5b18abb27a..941677a6b933 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -40,11 +40,13 @@ class HeuristicApplier(session: SparkSession) extends ColumnarRuleApplier with Logging with LogLevelUtil { + // This is an empirical value, may need to be changed for supporting other versions of spark. + private val aqeStackTraceIndex = 19 private lazy val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel private lazy val planChangeLogger = new PlanChangeLogger[SparkPlan]() - private val adaptiveContext = AdaptiveContext(session) + private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = withTransformRules(transformRules(outputsColumnar)).apply(plan) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala index 0592a3acafcc..4a9d69f8f0b1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala @@ -33,20 +33,19 @@ sealed trait AdaptiveContext { } object AdaptiveContext { - def apply(session: SparkSession): AdaptiveContext = new AdaptiveContextImpl(session) + def apply(session: SparkSession, aqeStackTraceIndex: Int): AdaptiveContext = + new AdaptiveContextImpl(session, aqeStackTraceIndex) private val GLUTEN_IS_ADAPTIVE_CONTEXT = "gluten.isAdaptiveContext" - // This is an empirical value, may need to be changed for supporting other versions of spark. - private val aqeStackTraceIndex = 19 - // Holds the original plan for possible entire fallback. private val localOriginalPlans: ThreadLocal[ListBuffer[SparkPlan]] = ThreadLocal.withInitial(() => ListBuffer.empty[SparkPlan]) private val localIsAdaptiveContextFlags: ThreadLocal[ListBuffer[Boolean]] = ThreadLocal.withInitial(() => ListBuffer.empty[Boolean]) - private class AdaptiveContextImpl(session: SparkSession) extends AdaptiveContext { + private class AdaptiveContextImpl(session: SparkSession, aqeStackTraceIndex: Int) + extends AdaptiveContext { // Just for test use. override def enableAdaptiveContext(): Unit = { session.sparkContext.setLocalProperty(GLUTEN_IS_ADAPTIVE_CONTEXT, "true") diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala index 98c4ca37c370..555e7d6143bc 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala @@ -16,27 +16,52 @@ */ package org.apache.gluten.planner -import org.apache.gluten.planner.cost.GlutenCostModel import org.apache.gluten.planner.metadata.GlutenMetadataModel import org.apache.gluten.planner.plan.GlutenPlanModel import org.apache.gluten.planner.property.GlutenPropertyModel -import org.apache.gluten.ras.{Optimization, RasExplain} +import org.apache.gluten.ras.{CostModel, Optimization, RasExplain} import org.apache.gluten.ras.rule.RasRule import org.apache.spark.sql.execution.SparkPlan +import scala.collection.mutable + object GlutenOptimization { + def builder(): Builder = new BuilderImpl + private object GlutenExplain extends RasExplain[SparkPlan] { override def describeNode(node: SparkPlan): String = node.nodeName } - def apply(rules: Seq[RasRule[SparkPlan]]): Optimization[SparkPlan] = { - Optimization[SparkPlan]( - GlutenPlanModel(), - GlutenCostModel(), - GlutenMetadataModel(), - GlutenPropertyModel(), - GlutenExplain, - RasRule.Factory.reuse(rules)) + trait Builder { + def addRules(rules: Seq[RasRule[SparkPlan]]): Builder + def costModel(costModel: CostModel[SparkPlan]): Builder + def create(): Optimization[SparkPlan] + } + + private class BuilderImpl extends Builder { + private val rules: mutable.ListBuffer[RasRule[SparkPlan]] = mutable.ListBuffer() + private var costModel: Option[CostModel[SparkPlan]] = None + + override def addRules(rules: Seq[RasRule[SparkPlan]]): Builder = { + this.rules ++= rules + this + } + + override def costModel(costModel: CostModel[SparkPlan]): Builder = { + this.costModel = Some(costModel) + this + } + + override def create(): Optimization[SparkPlan] = { + assert(costModel.isDefined, "Cost model is required to initialize GlutenOptimization") + Optimization[SparkPlan]( + GlutenPlanModel(), + costModel.get, + GlutenMetadataModel(), + GlutenPropertyModel(), + GlutenExplain, + RasRule.Factory.reuse(rules)) + } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala index fa69eedb5f23..4b5fa6803eec 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala @@ -16,23 +16,40 @@ */ package org.apache.gluten.planner.cost +import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.columnar.OffloadJoin +import org.apache.gluten.extension.columnar.enumerated.RemoveFilter import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike} import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Cost, CostModel} import org.apache.gluten.utils.PlanUtil +import org.apache.spark.internal.Logging import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan} import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec +import org.apache.spark.sql.utils.ReflectionUtil -class GlutenCostModel {} - -object GlutenCostModel { - def apply(): CostModel[SparkPlan] = { - RoughCostModel +object GlutenCostModel extends Logging { + def find(): CostModel[SparkPlan] = { + val aliases: Map[String, Class[_ <: CostModel[SparkPlan]]] = Map( + "rough" -> classOf[RoughCostModel]) + val aliasOrClass = GlutenConfig.getConf.rasCostModel + val clazz: Class[_ <: CostModel[SparkPlan]] = if (aliases.contains(aliasOrClass)) { + aliases(aliasOrClass) + } else { + val userModel = ReflectionUtil.classForName(aliasOrClass) + logInfo(s"Using user cost model: $aliasOrClass") + userModel + } + val ctor = clazz.getDeclaredConstructor() + ctor.setAccessible(true) + val model = ctor.newInstance() + model } - private object RoughCostModel extends CostModel[SparkPlan] { + def rough(): CostModel[SparkPlan] = new RoughCostModel() + + private class RoughCostModel extends CostModel[SparkPlan] { private val infLongCost = Long.MaxValue override def costOf(node: SparkPlan): GlutenCost = node match { @@ -62,6 +79,9 @@ object GlutenCostModel { // To exclude the rewritten intermediate plan that is not executable // by vanilla Spark and was generated by strategy "JoinSelectionOverrides" infLongCost + case _: RemoveFilter.NoopFilter => + // To make planner choose the tree that has applied rule PushFilterToScan. + 0L case ColumnarToRowExec(child) => 3L case RowToColumnarExec(child) => 3L case ColumnarToRowLike(child) => 3L diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala b/gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala new file mode 100644 index 000000000000..f674e7ec2e70 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/sql/utils/ReflectionUtil.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.utils + +import org.apache.spark.util.Utils + +object ReflectionUtil { + def classForName[C]( + className: String, + initialize: Boolean = true, + noSparkClassLoader: Boolean = false): Class[C] = { + Utils.classForName(className, initialize, noSparkClassLoader) + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 0fcaed8217ec..9db063a6623f 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -42,8 +42,6 @@ class GlutenConfig(conf: SQLConf) extends Logging { def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED) - def enableRas: Boolean = conf.getConf(RAS_ENABLED) - // FIXME the option currently controls both JVM and native validation against a Substrait plan. def enableNativeValidation: Boolean = conf.getConf(NATIVE_VALIDATION_ENABLED) @@ -251,6 +249,11 @@ class GlutenConfig(conf: SQLConf) extends Logging { def conservativeTaskOffHeapMemorySize: Long = conf.getConf(COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES) + // Options used by RAS. + def enableRas: Boolean = conf.getConf(RAS_ENABLED) + + def rasCostModel: String = conf.getConf(RAS_COST_MODEL) + def enableVeloxCache: Boolean = conf.getConf(COLUMNAR_VELOX_CACHE_ENABLED) def veloxMemCacheSize: Long = conf.getConf(COLUMNAR_VELOX_MEM_CACHE_SIZE) @@ -705,15 +708,6 @@ object GlutenConfig { .booleanConf .createWithDefault(GLUTEN_ENABLE_BY_DEFAULT) - val RAS_ENABLED = - buildConf("spark.gluten.sql.ras.enabled") - .doc( - "Experimental: Enables RAS (relational algebra selector) during physical " + - "planning to generate more efficient query plan. Note, this feature is still in " + - "development and may not bring performance profits.") - .booleanConf - .createWithDefault(false) - // FIXME the option currently controls both JVM and native validation against a Substrait plan. val NATIVE_VALIDATION_ENABLED = buildConf("spark.gluten.sql.enable.native.validation") @@ -1202,7 +1196,25 @@ object GlutenConfig { .bytesConf(ByteUnit.BYTE) .createWithDefaultString("8MB") - // velox caching options + // Options used by RAS. + val RAS_ENABLED = + buildConf("spark.gluten.ras.enabled") + .doc( + "Experimental: Enables RAS (relational algebra selector) during physical " + + "planning to generate more efficient query plan. Note, this feature is still in " + + "development and may not bring performance profits.") + .booleanConf + .createWithDefault(false) + + val RAS_COST_MODEL = + buildConf("spark.gluten.ras.costModel") + .doc( + "Experimental: The classpath of user-defined cost model that will be used by RAS. " + + "If not specified, a rough built-in cost model will be used.") + .stringConf + .createWithDefaultString("rough") + + // velox caching options. val COLUMNAR_VELOX_CACHE_ENABLED = buildStaticConf("spark.gluten.sql.columnar.backend.velox.cacheEnabled") .internal() From e1469f0c3859c9dd5f0edb9d2f5890794a540cee Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Mon, 27 May 2024 14:35:46 +0800 Subject: [PATCH 148/402] [VL] Daily Update Velox Version (2024_05_27) (#5872) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 6c25c8f08426..b71a7ad47473 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_24 +VELOX_BRANCH=2024_05_27 VELOX_HOME="" #Set on run gluten on HDFS From 280064a2e6bfa0924f9a2423c530b41e69d87cb9 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Mon, 27 May 2024 15:39:47 +0800 Subject: [PATCH 149/402] [VL] Allow hash on map for round robin repartitioning (#5349) --- .../velox/VeloxSparkPlanExecApi.scala | 49 +++++++++++++------ .../gluten/execution/TestOperator.scala | 16 ++++-- .../apache/gluten/fuzzer/FuzzerTestBase.scala | 2 +- .../fuzzer/ShuffleWriterFuzzerTest.scala | 2 +- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index cfa135046012..155a33c94a09 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -315,6 +315,16 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { override def genColumnarShuffleExchange( shuffle: ShuffleExchangeExec, newChild: SparkPlan): SparkPlan = { + def allowHashOnMap[T](f: => T): T = { + val originalAllowHash = SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE) + try { + SQLConf.get.setConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE, true) + f + } finally { + SQLConf.get.setConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE, originalAllowHash) + } + } + shuffle.outputPartitioning match { case HashPartitioning(exprs, _) => val hashExpr = new Murmur3Hash(exprs) @@ -331,21 +341,30 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { shuffle.withNewChildren(newChild :: Nil) } case RoundRobinPartitioning(num) if SQLConf.get.sortBeforeRepartition && num > 1 => - val hashExpr = new Murmur3Hash(newChild.output) - val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ newChild.output - val projectTransformer = ProjectExecTransformer(projectList, newChild) - val sortOrder = SortOrder(projectTransformer.output.head, Ascending) - val sortByHashCode = SortExecTransformer(Seq(sortOrder), global = false, projectTransformer) - val dropSortColumnTransformer = ProjectExecTransformer(projectList.drop(1), sortByHashCode) - val validationResult = dropSortColumnTransformer.doValidate() - if (validationResult.isValid) { - ColumnarShuffleExchangeExec( - shuffle, - dropSortColumnTransformer, - dropSortColumnTransformer.output) - } else { - TransformHints.tagNotTransformable(shuffle, validationResult) - shuffle.withNewChildren(newChild :: Nil) + // scalastyle:off line.size.limit + // Temporarily allow hash on map if it's disabled, otherwise HashExpression will fail to get + // resolved if its child contains map type. + // See https://github.com/apache/spark/blob/609bd4839e5d504917de74ed1cb9c23645fba51f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala#L279-L283 + // scalastyle:on line.size.limit + allowHashOnMap { + val hashExpr = new Murmur3Hash(newChild.output) + val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ newChild.output + val projectTransformer = ProjectExecTransformer(projectList, newChild) + val sortOrder = SortOrder(projectTransformer.output.head, Ascending) + val sortByHashCode = + SortExecTransformer(Seq(sortOrder), global = false, projectTransformer) + val dropSortColumnTransformer = + ProjectExecTransformer(projectList.drop(1), sortByHashCode) + val validationResult = dropSortColumnTransformer.doValidate() + if (validationResult.isValid) { + ColumnarShuffleExchangeExec( + shuffle, + dropSortColumnTransformer, + dropSortColumnTransformer.output) + } else { + TransformHints.tagNotTransformable(shuffle, validationResult) + shuffle.withNewChildren(newChild :: Nil) + } } case _ => ColumnarShuffleExchangeExec(shuffle, newChild, null) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 657039572d93..8e8423360d69 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -22,7 +22,7 @@ import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf -import org.apache.spark.sql.{AnalysisException, Row} +import org.apache.spark.sql.{AnalysisException, DataFrame, Row} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.functions._ @@ -1354,7 +1354,12 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } - test("test roundrobine with sort") { + test("test RoundRobin repartition with sort") { + def checkRoundRobinOperators(df: DataFrame): Unit = { + checkGlutenOperatorMatch[SortExecTransformer](df) + checkGlutenOperatorMatch[ColumnarShuffleExchangeExec](df) + } + // scalastyle:off runQueryAndCompare("SELECT /*+ REPARTITION(3) */ l_orderkey, l_partkey FROM lineitem") { /* @@ -1364,7 +1369,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite { +- ^(2) ProjectExecTransformer [hash(l_orderkey#16L, l_partkey#17L) AS hash_partition_key#302, l_orderkey#16L, l_partkey#17L] +- ^(2) BatchScanExecTransformer[l_orderkey#16L, l_partkey#17L] ParquetScan DataFilters: [], Format: parquet, Location: InMemoryFileIndex(1 paths)[..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct, PushedFilters: [] RuntimeFilters: [] */ - checkGlutenOperatorMatch[SortExecTransformer] + checkRoundRobinOperators } // scalastyle:on @@ -1377,6 +1382,11 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } } + + // Gluten-5206: test repartition on map type + runQueryAndCompare( + "SELECT /*+ REPARTITION(3) */ l_orderkey, map(l_orderkey, l_partkey) FROM lineitem")( + checkRoundRobinOperators) } test("Support Map type signature") { diff --git a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala index 7d59fbfae721..1ee79a2ade87 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala @@ -35,7 +35,7 @@ abstract class FuzzerTestBase extends VeloxWholeStageTransformerSuite { .set("spark.plugins", "org.apache.gluten.GlutenPlugin") .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") .set("spark.memory.offHeap.enabled", "true") - .set("spark.memory.offHeap.size", "512MB") + .set("spark.memory.offHeap.size", "4g") .set("spark.driver.memory", "4g") .set("spark.driver.maxResultSize", "4g") } diff --git a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala index 1d27f26811de..7d8fc56d9728 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala @@ -68,7 +68,7 @@ class ShuffleWriterFuzzerTest extends FuzzerTestBase { logWarning( s"==============================> " + s"Started reproduction (seed: ${dataGenerator.getSeed})") - val result = defaultRunner(testShuffle(sql)) + val result = defaultRunner(testShuffle(sql))() assert(result.isInstanceOf[Successful], s"Failed to run 'reproduce' with seed: $seed") } } From e128ba708156232e0a259b850f4822886a143fb7 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Mon, 27 May 2024 17:19:15 +0800 Subject: [PATCH 150/402] [VL] Enable soundex function (#5877) [VL] Enable soundex function. --- .../scala/org/apache/gluten/utils/CHExpressionUtil.scala | 1 + .../gluten/execution/ScalarFunctionsValidateSuite.scala | 6 ++++++ docs/velox-backend-support-progress.md | 2 +- .../org/apache/gluten/expression/ExpressionMappings.scala | 1 + .../org/apache/gluten/expression/ExpressionNames.scala | 1 + 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index 5f78d25cc5c2..94d2895943d4 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -184,6 +184,7 @@ object CHExpressionUtil { URL_DECODE -> DefaultValidator(), URL_ENCODE -> DefaultValidator(), SKEWNESS -> DefaultValidator(), + SOUNDEX -> DefaultValidator(), BIT_LENGTH -> DefaultValidator(), MAKE_YM_INTERVAL -> DefaultValidator(), MAP_ZIP_WITH -> DefaultValidator(), diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index f9ec07619281..5923d65032d0 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -643,6 +643,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("soundex") { + runQueryAndCompare("select soundex(c_comment) from customer limit 50") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("Test make_timestamp function") { withTempPath { path => diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index ccb253c24b48..05c76bc470a1 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -178,7 +178,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | rpad | rpad | | S | | | | | | | | | | | S | | | | | | | | | | rtrim | rtrim | rtrim | S | | | | | | | | | | | S | | | | | | | | | | sentences | | | | | | | | | | | | | | | | | | | | | | | -| soundex | | | | | | | | | | | | | | | | | | | | | | | +| soundex | | soundex | S | | | | | | | | | | | | | | | | | | | | | space | | | | | | | | | | | | | | | | | | | | | | | | split | split | split | S | Mismatched | | | | | | | | | | | | | | | | | | | | split_part | split_part | | | Mismatched | | | | | | | | | | | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 14371a71ecdc..1eade3da664a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -76,6 +76,7 @@ object ExpressionMappings { Sig[Length](LENGTH), Sig[Lower](LOWER), Sig[Upper](UPPER), + Sig[SoundEx](SOUNDEX), Sig[StringLocate](LOCATE), Sig[StringTrimLeft](LTRIM), Sig[StringTrimRight](RTRIM), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 6e6502c19b48..be7e32fc97d6 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -113,6 +113,7 @@ object ExpressionNames { final val LEFT = "left" final val REPEAT = "repeat" final val TRANSLATE = "translate" + final val SOUNDEX = "soundex" final val SPACE = "space" final val EMPTY2NULL = "empty2null" final val INITCAP = "initcap" From 48e05f780920a174842034f47b25921f8c3d2b93 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Mon, 27 May 2024 04:58:30 -0500 Subject: [PATCH 151/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240527) (#5871) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240527) * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/63620 * Support Sum0 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- .../local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 775667afb07d..b5d3aac8b42a 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240526 -CH_COMMIT=ff17e067fac \ No newline at end of file +CH_BRANCH=rebase_ch/20240527 +CH_COMMIT=dd16f9435bf diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp index c709a5f24cca..cdb7d3455680 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp @@ -352,7 +352,7 @@ void SparkMergeTreeWriter::writeTempPart(MergeTreeDataWriter::TemporaryPart & te /// This effectively chooses minimal compression method: /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_codec = storage->getContext()->chooseCompressionCodec(0, 0); - + auto txn = context->getCurrentTransaction(); auto out = std::make_unique( new_data_part, metadata_snapshot, @@ -360,7 +360,7 @@ void SparkMergeTreeWriter::writeTempPart(MergeTreeDataWriter::TemporaryPart & te indices, MergeTreeStatisticsFactory::instance().getMany(metadata_snapshot->getColumns()), compression_codec, - context->getCurrentTransaction(), + txn ? txn->tid : Tx::PrehistoricTID, false, false, context->getWriteSettings()); From 95096e3d97277fa93b3ef7d8a921dd67d3422b6a Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Mon, 27 May 2024 18:23:39 +0800 Subject: [PATCH 152/402] [VL] Enable arrays_overlap function (#5878) [VL] Enable arrays_overlap function. --- .../execution/ScalarFunctionsValidateSuite.scala | 16 ++++++++++++++++ .../substrait/SubstraitToVeloxPlanValidator.cc | 1 - docs/velox-backend-support-progress.md | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 5923d65032d0..8802c61c5f04 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -1037,4 +1037,20 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } } + + test("arrays_overlap") { + withTempPath { + path => + Seq[(Seq[Integer], Seq[Integer])]((Seq(1, 2, 3), Seq(3, 4)), (Seq(5, null), Seq())) + .toDF("v1", "v2") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl") + + runQueryAndCompare("select arrays_overlap(v1, v2) from array_tbl;") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } } diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index 0b08ca20517b..abb2bbc560f4 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -68,7 +68,6 @@ static const std::unordered_set kBlackList = { "repeat", "trunc", "sequence", - "arrays_overlap", "approx_percentile", "get_array_struct_fields"}; diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 05c76bc470a1..5d083c4e59ba 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -272,7 +272,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | array_repeat | | | S | | S | S | S | S | S | S | S | S | S | S | S | | | | | | | | | array_sort | array_sort | array_sort | S | | | | | | | | | | | | | | | | | | | | | array_union | | | | | | | | | | | | | | | | | | | | | | | -| arrays_overlap | array_overlap | | | | | | | | | | | | | | | | | | | | | | +| arrays_overlap | array_overlap | S | | | | | | | | | | | | | | | | | | | | | | arrays_zip | zip | | S | | | | | | | | | | | | | | | | | | | | | cardinality | cardinality | | | | | | | | | | | | | | | | | | | | | | | element_at | element_at | element_at | S | | | | | | | | | | | | | | | | S | S | | | From efd6f31fb44ea21846c0292d5ea31a3e05aa3af3 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 27 May 2024 20:44:36 +0800 Subject: [PATCH 153/402] [VL] Support DecimalType for approx_count_distinct (#5868) [VL] Support DecimalType for approx_count_distinct. --- .../gluten/extension/HLLRewriteRule.scala | 1 + .../VeloxAggregateFunctionsSuite.scala | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala index cb1e626a1ea6..03819fc102ab 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala @@ -73,6 +73,7 @@ case class HLLRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { case LongType => true case ShortType => true case StringType => true + case _: DecimalType => true case _ => false } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index faa361edf5aa..ffed6373123e 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -571,6 +571,26 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu } } + test("approx_count_distinct decimal") { + // The data type of l_discount is decimal. + runQueryAndCompare(""" + |select approx_count_distinct(l_discount) from lineitem; + |""".stripMargin) { + checkGlutenOperatorMatch[HashAggregateExecTransformer] + } + runQueryAndCompare( + "select approx_count_distinct(l_discount), count(distinct l_orderkey) from lineitem") { + df => + { + assert( + getExecutedPlan(df).count( + plan => { + plan.isInstanceOf[HashAggregateExecTransformer] + }) == 0) + } + } + } + test("max_by") { runQueryAndCompare(s""" |select max_by(l_linenumber, l_comment) from lineitem; From 4592e07fdb4179c68e41405b020e7cae202d7336 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Tue, 28 May 2024 08:21:26 +0800 Subject: [PATCH 154/402] [CORE] Avoid copy in ByteLiteralNode (#5763) Optimize the serialized bloom filter to avoid extra copy --- .../substrait/expression/BinaryLiteralNode.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/expression/BinaryLiteralNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/expression/BinaryLiteralNode.java index 864aeb741a15..3d1ee51741ee 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/expression/BinaryLiteralNode.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/expression/BinaryLiteralNode.java @@ -22,6 +22,9 @@ import com.google.protobuf.ByteString; import io.substrait.proto.Expression.Literal.Builder; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + public class BinaryLiteralNode extends LiteralNodeWithValue { public BinaryLiteralNode(byte[] value) { super(value, new BinaryTypeNode(true)); @@ -33,6 +36,14 @@ public BinaryLiteralNode(byte[] value, TypeNode typeNode) { @Override protected void updateLiteralBuilder(Builder literalBuilder, byte[] value) { - literalBuilder.setBinary(ByteString.copyFrom(value)); + ByteString byteValue; + try { + Method m = ByteString.class.getDeclaredMethod("wrap", byte[].class); + m.setAccessible(true); + byteValue = (ByteString) m.invoke(null, value); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + literalBuilder.setBinary(byteValue); } } From 50cd286e33cc06b8b6d991335cb44fa6598f6e2e Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Tue, 28 May 2024 08:30:07 +0800 Subject: [PATCH 155/402] [INFRA] Do not require all conversations resolved (#5865) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index ae4827046242..3a67e45bfdbe 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -38,7 +38,7 @@ github: required_approving_review_count: 1 required_signatures: false required_linear_history: true - required_conversation_resolution: true + required_conversation_resolution: false features: issues: true discussions: true From 12fdb7029a4921ff035c8a979c1580447d0acb72 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 28 May 2024 11:11:19 +0800 Subject: [PATCH 156/402] [VL] Following #5861, append some nit changes --- .../gluten/extension/columnar/enumerated/RemoveFilter.scala | 4 ++-- .../org/apache/gluten/planner/cost/GlutenCostModel.scala | 2 +- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index a3b4831a622d..55b29cd56ff1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -54,12 +54,12 @@ object RemoveFilter extends RasRule[SparkPlan] { leaf(clazz(classOf[BasicScanExecTransformer])) ).build()) - // A noop filter placeholder that indicates that all conditions are pushed into scan. + // A noop filter placeholder that indicates that all conditions are pushed down to scan. // // This operator has zero cost in cost model to avoid planner from choosing the // original filter-scan that doesn't have all conditions pushed down to scan. // - // We cannot simplify remove the filter to let planner choose the scan since by vanilla + // We cannot simply remove the filter to let planner choose the pushed scan since by vanilla // Spark's definition the filter may have different output nullability than scan. So // we have to keep this empty filter to let the optimized tree have the identical output schema // with the original tree. If we simply remove the filter, possible UBs might be caused. For diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala index 4b5fa6803eec..c45314a9f58f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala @@ -80,7 +80,7 @@ object GlutenCostModel extends Logging { // by vanilla Spark and was generated by strategy "JoinSelectionOverrides" infLongCost case _: RemoveFilter.NoopFilter => - // To make planner choose the tree that has applied rule PushFilterToScan. + // To make planner choose the tree that has applied rule PushFilterToScan. 0L case ColumnarToRowExec(child) => 3L case RowToColumnarExec(child) => 3L diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 9db063a6623f..d353c75c3a51 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -1209,7 +1209,7 @@ object GlutenConfig { val RAS_COST_MODEL = buildConf("spark.gluten.ras.costModel") .doc( - "Experimental: The classpath of user-defined cost model that will be used by RAS. " + + "Experimental: The class name of user-defined cost model that will be used by RAS. " + "If not specified, a rough built-in cost model will be used.") .stringConf .createWithDefaultString("rough") From 947f28a7aab150f159f3277928a39bd84f1bc9e1 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Tue, 28 May 2024 14:05:07 +0800 Subject: [PATCH 157/402] [CH] Add Compatibility test found by internal (#5882) Add Compatibility test found by internal --- .../compatibility/GlutenFunctionSuite.scala | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala new file mode 100644 index 000000000000..d0e13b49609a --- /dev/null +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.gluten.compatibility + +import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite +import org.apache.gluten.utils.UTSystemParameters + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.gluten.test.GlutenSQLTestUtils + +class GlutenFunctionSuite + extends GlutenClickHouseWholeStageTransformerSuite + with GlutenSQLTestUtils + with Logging { + + override protected val fileFormat: String = "parquet" + private val testPath: String = s"${UTSystemParameters.testDataPath}/$fileFormat/function" + + case class TestCase( + name: String, + sql: String, + ignore: Boolean = false + ) + + private val testCase = Seq( + TestCase( + "left", + s"""|select + | left(`99`, 2) + | , left(`100`, 3) + | , left(`101`, 4) + | , left(`101`, 0) + | , left(`101`, -1) -- error + | from parquet.`$testPath/left` + |""".stripMargin, + ignore = true + ), + TestCase( + "trim", + s"""|select + | trim(both ' ' from `99`) + | , trim(LEADING `100` from `99`) -- error + | , trim(TRAILING `100` from `99`) -- error + | from parquet.`$testPath/left` + |""".stripMargin, + ignore = true + ), + TestCase( + "date_format 1", + s"""|select + | `0` + | , date_format(`0`, 'y') + | , date_format(`0`, 'M') + | , date_format(`0`, 'D') -- error timezone related issue + | , date_format(`0`, 'd') + | , date_format(`0`, 'H') + | , date_format(`0`, 'h') + | , date_format(`0`, 'm') + | , date_format(`0`, 's') + | from parquet.`$testPath/date_format/date` + |""".stripMargin, + ignore = true + ), + TestCase( + "date_format 2", + s"""|select + | `4` + | , date_format(`4`, 'y') + | , date_format(`4`, 'M') + | , date_format(`4`, 'D') -- error timezone related issue + | , date_format(`4`, 'd') + | , date_format(`4`, 'H') + | , date_format(`4`, 'h') + | , date_format(`4`, 'm') + | , date_format(`4`, 's') + | from parquet.`$testPath/date_format/timestamp` + |""".stripMargin, + ignore = true + ) + ) + + testCase.foreach { + data => + if (data.ignore) { + ignore(s"${data.name}") {} + } else { + test(s"${data.name}") { + compareResultsAgainstVanillaSpark( + data.sql, + compareResult = true, + { _ => } + ) + } + } + } + +} From 291f0842905b704db6f303e88295280e383d0a38 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 28 May 2024 14:25:38 +0800 Subject: [PATCH 158/402] [GLUTEN-5840][VL] Fix udaf register simple intermediate type (#5876) --- .../spark/sql/expression/UDFResolver.scala | 14 +- .../gluten/expression/VeloxUdfSuite.scala | 5 +- cpp/velox/udf/examples/MyUDAF.cc | 366 +++++++++++++++--- cpp/velox/udf/examples/MyUDF.cc | 30 +- cpp/velox/udf/examples/UdfCommon.h | 53 +++ docs/developers/VeloxUDF.md | 7 +- 6 files changed, 380 insertions(+), 95 deletions(-) create mode 100644 cpp/velox/udf/examples/UdfCommon.h diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index ec98e98f1c6e..915fc554584c 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -175,12 +175,16 @@ object UDFResolver extends Logging { intermediateTypes: ExpressionType, variableArity: Boolean): Unit = { assert(argTypes.dataType.isInstanceOf[StructType]) - assert(intermediateTypes.dataType.isInstanceOf[StructType]) - val aggBufferAttributes = - intermediateTypes.dataType.asInstanceOf[StructType].fields.zipWithIndex.map { - case (f, index) => - AttributeReference(s"inter_$index", f.dataType, f.nullable)() + val aggBufferAttributes: Seq[AttributeReference] = + intermediateTypes.dataType match { + case StructType(fields) => + fields.zipWithIndex.map { + case (f, index) => + AttributeReference(s"agg_inter_$index", f.dataType, f.nullable)() + } + case t => + Seq(AttributeReference(s"agg_inter", t)()) } val v = diff --git a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala index 4d2f9fae3147..534a8d9f1c74 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/expression/VeloxUdfSuite.scala @@ -93,12 +93,13 @@ abstract class VeloxUdfSuite extends GlutenQueryTest with SQLHelper { | myavg(1), | myavg(1L), | myavg(cast(1.0 as float)), - | myavg(cast(1.0 as double)) + | myavg(cast(1.0 as double)), + | mycount_if(true) |""".stripMargin) df.collect() assert( df.collect() - .sameElements(Array(Row(1.0, 1.0, 1.0, 1.0)))) + .sameElements(Array(Row(1.0, 1.0, 1.0, 1.0, 1L)))) } } diff --git a/cpp/velox/udf/examples/MyUDAF.cc b/cpp/velox/udf/examples/MyUDAF.cc index e6c4b1fea7e0..710bce53ae65 100644 --- a/cpp/velox/udf/examples/MyUDAF.cc +++ b/cpp/velox/udf/examples/MyUDAF.cc @@ -20,19 +20,22 @@ #include #include #include -#include + #include "udf/Udaf.h" +#include "udf/examples/UdfCommon.h" using namespace facebook::velox; using namespace facebook::velox::exec; namespace { +static const char* kBoolean = "boolean"; static const char* kInteger = "int"; static const char* kBigInt = "bigint"; static const char* kFloat = "float"; static const char* kDouble = "double"; +namespace myavg { // Copied from velox/exec/tests/SimpleAverageAggregate.cpp // Implementation of the average aggregation function through the @@ -98,84 +101,321 @@ class AverageAggregate { }; }; -exec::AggregateRegistrationResult registerSimpleAverageAggregate(const std::string& name) { - std::vector> signatures; +class MyAvgRegisterer final : public gluten::UdafRegisterer { + int getNumUdaf() override { + return 4; + } + + void populateUdafEntries(int& index, gluten::UdafEntry* udafEntries) override { + for (const auto& argTypes : {myAvgArg1_, myAvgArg2_, myAvgArg3_, myAvgArg4_}) { + udafEntries[index++] = {name_.c_str(), kDouble, 1, argTypes, myAvgIntermediateType_}; + } + } + + void registerSignatures() override { + registerSimpleAverageAggregate(); + } + + private: + exec::AggregateRegistrationResult registerSimpleAverageAggregate() { + std::vector> signatures; + + for (const auto& inputType : {"smallint", "integer", "bigint", "double"}) { + signatures.push_back(exec::AggregateFunctionSignatureBuilder() + .returnType("double") + .intermediateType("row(double,bigint)") + .argumentType(inputType) + .build()); + } - for (const auto& inputType : {"smallint", "integer", "bigint", "double"}) { signatures.push_back(exec::AggregateFunctionSignatureBuilder() - .returnType("double") + .returnType("real") .intermediateType("row(double,bigint)") - .argumentType(inputType) + .argumentType("real") .build()); - } - signatures.push_back(exec::AggregateFunctionSignatureBuilder() - .returnType("real") - .intermediateType("row(double,bigint)") - .argumentType("real") - .build()); - - return exec::registerAggregateFunction( - name, - std::move(signatures), - [name]( - core::AggregationNode::Step step, - const std::vector& argTypes, - const TypePtr& resultType, - const core::QueryConfig& /*config*/) -> std::unique_ptr { - VELOX_CHECK_LE(argTypes.size(), 1, "{} takes at most one argument", name); - auto inputType = argTypes[0]; - if (exec::isRawInput(step)) { - switch (inputType->kind()) { - case TypeKind::SMALLINT: - return std::make_unique>>(resultType); - case TypeKind::INTEGER: - return std::make_unique>>(resultType); - case TypeKind::BIGINT: - return std::make_unique>>(resultType); - case TypeKind::REAL: - return std::make_unique>>(resultType); - case TypeKind::DOUBLE: - return std::make_unique>>(resultType); - default: - VELOX_FAIL("Unknown input type for {} aggregation {}", name, inputType->kindName()); - } - } else { - switch (resultType->kind()) { - case TypeKind::REAL: - return std::make_unique>>(resultType); - case TypeKind::DOUBLE: - case TypeKind::ROW: - return std::make_unique>>(resultType); - default: - VELOX_FAIL("Unsupported result type for final aggregation: {}", resultType->kindName()); + return exec::registerAggregateFunction( + name_, + std::move(signatures), + [this]( + core::AggregationNode::Step step, + const std::vector& argTypes, + const TypePtr& resultType, + const core::QueryConfig& /*config*/) -> std::unique_ptr { + VELOX_CHECK_LE(argTypes.size(), 1, "{} takes at most one argument", name_); + auto inputType = argTypes[0]; + if (exec::isRawInput(step)) { + switch (inputType->kind()) { + case TypeKind::SMALLINT: + return std::make_unique>>(resultType); + case TypeKind::INTEGER: + return std::make_unique>>(resultType); + case TypeKind::BIGINT: + return std::make_unique>>(resultType); + case TypeKind::REAL: + return std::make_unique>>(resultType); + case TypeKind::DOUBLE: + return std::make_unique>>(resultType); + default: + VELOX_FAIL("Unknown input type for {} aggregation {}", name_, inputType->kindName()); + } + } else { + switch (resultType->kind()) { + case TypeKind::REAL: + return std::make_unique>>(resultType); + case TypeKind::DOUBLE: + case TypeKind::ROW: + return std::make_unique>>(resultType); + default: + VELOX_FAIL("Unsupported result type for final aggregation: {}", resultType->kindName()); + } } + }, + true /*registerCompanionFunctions*/, + true /*overwrite*/); + } + + const std::string name_ = "myavg"; + const char* myAvgArg1_[1] = {kInteger}; + const char* myAvgArg2_[1] = {kBigInt}; + const char* myAvgArg3_[1] = {kFloat}; + const char* myAvgArg4_[1] = {kDouble}; + + const char* myAvgIntermediateType_ = "struct"; +}; +} // namespace myavg + +namespace mycountif { + +// Copied from velox/functions/prestosql/aggregates/CountIfAggregate.cpp +class CountIfAggregate : public exec::Aggregate { + public: + explicit CountIfAggregate() : exec::Aggregate(BIGINT()) {} + + int32_t accumulatorFixedWidthSize() const override { + return sizeof(int64_t); + } + + void extractAccumulators(char** groups, int32_t numGroups, VectorPtr* result) override { + extractValues(groups, numGroups, result); + } + + void extractValues(char** groups, int32_t numGroups, VectorPtr* result) override { + auto* vector = (*result)->as>(); + VELOX_CHECK(vector); + vector->resize(numGroups); + + auto* rawValues = vector->mutableRawValues(); + for (vector_size_t i = 0; i < numGroups; ++i) { + rawValues[i] = *value(groups[i]); + } + } + + void addRawInput( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + DecodedVector decoded(*args[0], rows); + + if (decoded.isConstantMapping()) { + if (decoded.isNullAt(0)) { + return; + } + if (decoded.valueAt(0)) { + rows.applyToSelected([&](vector_size_t i) { addToGroup(groups[i], 1); }); + } + } else if (decoded.mayHaveNulls()) { + rows.applyToSelected([&](vector_size_t i) { + if (decoded.isNullAt(i)) { + return; + } + if (decoded.valueAt(i)) { + addToGroup(groups[i], 1); } - }, - true /*registerCompanionFunctions*/, - true /*overwrite*/); + }); + } else { + rows.applyToSelected([&](vector_size_t i) { + if (decoded.valueAt(i)) { + addToGroup(groups[i], 1); + } + }); + } + } + + void addIntermediateResults( + char** groups, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + DecodedVector decoded(*args[0], rows); + + if (decoded.isConstantMapping()) { + auto numTrue = decoded.valueAt(0); + rows.applyToSelected([&](vector_size_t i) { addToGroup(groups[i], numTrue); }); + return; + } + + rows.applyToSelected([&](vector_size_t i) { + auto numTrue = decoded.valueAt(i); + addToGroup(groups[i], numTrue); + }); + } + + void addSingleGroupRawInput( + char* group, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + DecodedVector decoded(*args[0], rows); + + // Constant mapping - check once and add number of selected rows if true. + if (decoded.isConstantMapping()) { + if (!decoded.isNullAt(0)) { + auto isTrue = decoded.valueAt(0); + if (isTrue) { + addToGroup(group, rows.countSelected()); + } + } + return; + } + + int64_t numTrue = 0; + if (decoded.mayHaveNulls()) { + rows.applyToSelected([&](vector_size_t i) { + if (decoded.isNullAt(i)) { + return; + } + if (decoded.valueAt(i)) { + ++numTrue; + } + }); + } else { + rows.applyToSelected([&](vector_size_t i) { + if (decoded.valueAt(i)) { + ++numTrue; + } + }); + } + addToGroup(group, numTrue); + } + + void addSingleGroupIntermediateResults( + char* group, + const SelectivityVector& rows, + const std::vector& args, + bool /*mayPushdown*/) override { + auto arg = args[0]->as>(); + + int64_t numTrue = 0; + rows.applyToSelected([&](auto row) { numTrue += arg->valueAt(row); }); + + addToGroup(group, numTrue); + } + + protected: + void initializeNewGroupsInternal(char** groups, folly::Range indices) override { + for (auto i : indices) { + *value(groups[i]) = 0; + } + } + + private: + inline void addToGroup(char* group, int64_t numTrue) { + *value(group) += numTrue; + } +}; + +class MyCountIfRegisterer final : public gluten::UdafRegisterer { + int getNumUdaf() override { + return 1; + } + + void populateUdafEntries(int& index, gluten::UdafEntry* udafEntries) override { + udafEntries[index++] = {name_.c_str(), kBigInt, 1, myCountIfArg_, kBigInt}; + } + + void registerSignatures() override { + registerCountIfAggregate(); + } + + private: + void registerCountIfAggregate() { + std::vector> signatures{ + exec::AggregateFunctionSignatureBuilder() + .returnType("bigint") + .intermediateType("bigint") + .argumentType("boolean") + .build(), + }; + + exec::registerAggregateFunction( + name_, + std::move(signatures), + [this]( + core::AggregationNode::Step step, + std::vector argTypes, + const TypePtr& /*resultType*/, + const core::QueryConfig& /*config*/) -> std::unique_ptr { + VELOX_CHECK_EQ(argTypes.size(), 1, "{} takes one argument", name_); + + auto isPartial = exec::isRawInput(step); + if (isPartial) { + VELOX_CHECK_EQ(argTypes[0]->kind(), TypeKind::BOOLEAN, "{} function only accepts boolean parameter", name_); + } + + return std::make_unique(); + }, + {false /*orderSensitive*/}, + true, + true); + } + + const std::string name_ = "mycount_if"; + const char* myCountIfArg_[1] = {kBoolean}; +}; +} // namespace mycountif + +std::vector>& globalRegisters() { + static std::vector> registerers; + return registerers; } -} // namespace -const int kNumMyUdaf = 4; +void setupRegisterers() { + static bool inited = false; + if (inited) { + return; + } + auto& registerers = globalRegisters(); + registerers.push_back(std::make_shared()); + registerers.push_back(std::make_shared()); + inited = true; +} +} // namespace DEFINE_GET_NUM_UDAF { - return kNumMyUdaf; + setupRegisterers(); + + int numUdf = 0; + for (const auto& registerer : globalRegisters()) { + numUdf += registerer->getNumUdaf(); + } + return numUdf; } -const char* myAvgArg1[] = {kInteger}; -const char* myAvgArg2[] = {kBigInt}; -const char* myAvgArg3[] = {kFloat}; -const char* myAvgArg4[] = {kDouble}; -const char* myAvgIntermediateType = "struct"; DEFINE_GET_UDAF_ENTRIES { + setupRegisterers(); + int index = 0; - udafEntries[index++] = {"myavg", kDouble, 1, myAvgArg1, myAvgIntermediateType}; - udafEntries[index++] = {"myavg", kDouble, 1, myAvgArg2, myAvgIntermediateType}; - udafEntries[index++] = {"myavg", kDouble, 1, myAvgArg3, myAvgIntermediateType}; - udafEntries[index++] = {"myavg", kDouble, 1, myAvgArg4, myAvgIntermediateType}; + for (const auto& registerer : globalRegisters()) { + registerer->populateUdafEntries(index, udafEntries); + } } DEFINE_REGISTER_UDAF { - registerSimpleAverageAggregate("myavg"); + setupRegisterers(); + + for (const auto& registerer : globalRegisters()) { + registerer->registerSignatures(); + } } diff --git a/cpp/velox/udf/examples/MyUDF.cc b/cpp/velox/udf/examples/MyUDF.cc index 88bc3ad85da3..ee20ca39d026 100644 --- a/cpp/velox/udf/examples/MyUDF.cc +++ b/cpp/velox/udf/examples/MyUDF.cc @@ -20,28 +20,17 @@ #include #include #include "udf/Udf.h" +#include "udf/examples/UdfCommon.h" using namespace facebook::velox; using namespace facebook::velox::exec; +namespace { + static const char* kInteger = "int"; static const char* kBigInt = "bigint"; static const char* kDate = "date"; -class UdfRegisterer { - public: - ~UdfRegisterer() = default; - - // Returns the number of UDFs in populateUdfEntries. - virtual int getNumUdf() = 0; - - // Populate the udfEntries, starting at the given index. - virtual void populateUdfEntries(int& index, gluten::UdfEntry* udfEntries) = 0; - - // Register all function signatures to velox. - virtual void registerSignatures() = 0; -}; - namespace myudf { template @@ -106,7 +95,7 @@ static std::shared_ptr makePlusConstant( // signatures: // bigint -> bigint // type: VectorFunction -class MyUdf1Registerer final : public UdfRegisterer { +class MyUdf1Registerer final : public gluten::UdfRegisterer { public: int getNumUdf() override { return 1; @@ -135,7 +124,7 @@ class MyUdf1Registerer final : public UdfRegisterer { // integer -> integer // bigint -> bigint // type: StatefulVectorFunction -class MyUdf2Registerer final : public UdfRegisterer { +class MyUdf2Registerer final : public gluten::UdfRegisterer { public: int getNumUdf() override { return 2; @@ -167,7 +156,7 @@ class MyUdf2Registerer final : public UdfRegisterer { // [integer,] ... -> integer // bigint, [bigint,] ... -> bigint // type: StatefulVectorFunction with variable arity -class MyUdf3Registerer final : public UdfRegisterer { +class MyUdf3Registerer final : public gluten::UdfRegisterer { public: int getNumUdf() override { return 2; @@ -215,7 +204,7 @@ struct MyDateSimpleFunction { // signatures: // date, integer -> bigint // type: SimpleFunction -class MyDateRegisterer final : public UdfRegisterer { +class MyDateRegisterer final : public gluten::UdfRegisterer { public: int getNumUdf() override { return 1; @@ -235,8 +224,8 @@ class MyDateRegisterer final : public UdfRegisterer { }; } // namespace mydate -std::vector>& globalRegisters() { - static std::vector> registerers; +std::vector>& globalRegisters() { + static std::vector> registerers; return registerers; } @@ -252,6 +241,7 @@ void setupRegisterers() { registerers.push_back(std::make_shared()); inited = true; } +} // namespace DEFINE_GET_NUM_UDF { setupRegisterers(); diff --git a/cpp/velox/udf/examples/UdfCommon.h b/cpp/velox/udf/examples/UdfCommon.h new file mode 100644 index 000000000000..a68c474607cd --- /dev/null +++ b/cpp/velox/udf/examples/UdfCommon.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "udf/Udaf.h" +#include "udf/Udf.h" + +namespace gluten { + +class UdfRegisterer { + public: + ~UdfRegisterer() = default; + + // Returns the number of UDFs in populateUdfEntries. + virtual int getNumUdf() = 0; + + // Populate the udfEntries, starting at the given index. + virtual void populateUdfEntries(int& index, gluten::UdfEntry* udfEntries) = 0; + + // Register all function signatures to velox. + virtual void registerSignatures() = 0; +}; + +class UdafRegisterer { + public: + ~UdafRegisterer() = default; + + // Returns the number of UDFs in populateUdafEntries. + virtual int getNumUdaf() = 0; + + // Populate the udfEntries, starting at the given index. + virtual void populateUdafEntries(int& index, gluten::UdafEntry* udafEntries) = 0; + + // Register all function signatures to velox. + virtual void registerSignatures() = 0; +}; + +} // namespace gluten \ No newline at end of file diff --git a/docs/developers/VeloxUDF.md b/docs/developers/VeloxUDF.md index b88c4de1515e..c896fd672657 100644 --- a/docs/developers/VeloxUDF.md +++ b/docs/developers/VeloxUDF.md @@ -137,13 +137,10 @@ You can also specify the local or HDFS URIs to the UDF libraries or archives. Lo ## Try the example We provided Velox UDF examples in file [MyUDF.cc](../../cpp/velox/udf/examples/MyUDF.cc) and UDAF examples in file [MyUDAF.cc](../../cpp/velox/udf/examples/MyUDAF.cc). -You need to build the gluten cpp project with `--build_example=ON` to get the example libraries. +You need to build the gluten project with `--build_example=ON` to get the example libraries. ```shell -## compile Gluten cpp module -cd /path/to/gluten/cpp -## if you use custom velox_home, make sure specified here by --velox_home -./compile.sh --build_velox_backend=ON --build_examples=ON +./dev/buildbundle-veloxbe.sh --build_examples=ON ``` Then, you can find the example libraries at /path/to/gluten/cpp/build/velox/udf/examples/ From 729d3450e0e853b6647079452f7051c642756a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Tue, 28 May 2024 14:42:29 +0800 Subject: [PATCH 159/402] [VL][Core] SampleExec Operator Native Support (#5856) [VL] SampleExec Operator Native Support. --- .../backendsapi/clickhouse/CHMetricsApi.scala | 11 ++ .../clickhouse/CHSparkPlanExecApi.scala | 8 ++ .../backendsapi/velox/VeloxBackend.scala | 2 + .../backendsapi/velox/VeloxMetricsApi.scala | 16 +++ .../velox/VeloxSparkPlanExecApi.scala | 9 ++ .../gluten/execution/TestOperator.scala | 12 ++ .../backendsapi/BackendSettingsApi.scala | 2 + .../gluten/backendsapi/MetricsApi.scala | 4 + .../gluten/backendsapi/SparkPlanExecApi.scala | 7 + .../execution/SampleExecTransformer.scala | 126 ++++++++++++++++++ .../columnar/OffloadSingleNode.scala | 9 ++ .../columnar/TransformHintRule.scala | 9 ++ .../columnar/validator/Validators.scala | 3 + .../gluten/metrics/SampleMetricsUpdater.scala | 35 +++++ .../org/apache/gluten/GlutenConfig.scala | 9 ++ 15 files changed, 262 insertions(+) create mode 100644 gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala create mode 100644 gluten-data/src/main/scala/org/apache/gluten/metrics/SampleMetricsUpdater.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala index 30f682f0fb2e..350548e981d6 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala @@ -361,6 +361,17 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { s"NestedLoopJoinTransformer metrics update is not supported in CH backend") } + override def genSampleTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = { + throw new UnsupportedOperationException( + s"SampleTransformer metrics update is not supported in CH backend") + } + + override def genSampleTransformerMetricsUpdater( + metrics: Map[String, SQLMetric]): MetricsUpdater = { + throw new UnsupportedOperationException( + s"SampleTransformer metrics update is not supported in CH backend") + } + def genWriteFilesTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = { throw new UnsupportedOperationException( s"WriteFilesTransformer metrics update is not supported in CH backend") diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 8c2b20db6f84..1403c8261df8 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -379,6 +379,14 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { throw new GlutenNotSupportException( "BroadcastNestedLoopJoinExecTransformer is not supported in ch backend.") + override def genSampleExecTransformer( + lowerBound: Double, + upperBound: Double, + withReplacement: Boolean, + seed: Long, + child: SparkPlan): SampleExecTransformer = + throw new GlutenNotSupportException("SampleExecTransformer is not supported in ch backend.") + /** Generate an expression transformer to transform GetMapValue to Substrait. */ def genGetMapValueTransformer( substraitExprName: String, diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index a2da0b8b2a86..7f928bd330b7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -499,6 +499,8 @@ object VeloxBackendSettings extends BackendSettingsApi { override def supportBroadcastNestedLoopJoinExec(): Boolean = true + override def supportSampleExec(): Boolean = true + override def supportColumnarArrowUdf(): Boolean = true override def generateHdfsConfForLibhdfs(): Boolean = true diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala index 7be639d4caf5..0811d71d16b8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala @@ -540,4 +540,20 @@ class VeloxMetricsApi extends MetricsApi with Logging { override def genNestedLoopJoinTransformerMetricsUpdater( metrics: Map[String, SQLMetric]): MetricsUpdater = new NestedLoopJoinMetricsUpdater(metrics) + + override def genSampleTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = + Map( + "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), + "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), + "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of sample"), + "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), + "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), + "numMemoryAllocations" -> SQLMetrics.createMetric( + sparkContext, + "number of memory allocations") + ) + + override def genSampleTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = + new SampleMetricsUpdater(metrics) } diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 155a33c94a09..92be63a583f6 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -411,6 +411,15 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { right, isNullAwareAntiJoin) + override def genSampleExecTransformer( + lowerBound: Double, + upperBound: Double, + withReplacement: Boolean, + seed: Long, + child: SparkPlan): SampleExecTransformer = { + SampleExecTransformer(lowerBound, upperBound, withReplacement, seed, child) + } + override def genSortMergeJoinExecTransformer( leftKeys: Seq[Expression], rightKeys: Seq[Expression], diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 8e8423360d69..7bbc24d45b6d 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1050,6 +1050,18 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + test("Test sample op") { + withSQLConf("spark.gluten.sql.columnarSampleEnabled" -> "true") { + withTable("t") { + sql("create table t (id int, b boolean) using parquet") + sql("insert into t values (1, true), (2, false), (3, null), (4, true), (5, false)") + runQueryAndCompare("select * from t TABLESAMPLE(20 PERCENT)", false) { + checkGlutenOperatorMatch[SampleExecTransformer] + } + } + } + } + test("test cross join") { withTable("t1", "t2") { sql(""" diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index 9c5c13271aeb..d18273af2faa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -144,6 +144,8 @@ trait BackendSettingsApi { def supportBroadcastNestedLoopJoinExec(): Boolean = false + def supportSampleExec(): Boolean = false + /** Merge two phases hash based aggregate if need */ def mergeTwoPhasesHashBaseAggregateIfNeed(): Boolean = false diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala index 99b44a2de350..a96f27f5a8a3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/MetricsApi.scala @@ -113,6 +113,10 @@ trait MetricsApi extends Serializable { def genNestedLoopJoinTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater + def genSampleTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] + + def genSampleTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater + def genColumnarInMemoryTableMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows")) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 429b926cdceb..78cf02f0ac24 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -125,6 +125,13 @@ trait SparkPlanExecApi { right: SparkPlan, isNullAwareAntiJoin: Boolean = false): BroadcastHashJoinExecTransformerBase + def genSampleExecTransformer( + lowerBound: Double, + upperBound: Double, + withReplacement: Boolean, + seed: Long, + child: SparkPlan): SampleExecTransformer + /** Generate ShuffledHashJoinExecTransformer. */ def genSortMergeJoinExecTransformer( leftKeys: Seq[Expression], diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala new file mode 100644 index 000000000000..86189392af75 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.expression.{ConverterUtils, ExpressionConverter} +import org.apache.gluten.extension.ValidationResult +import org.apache.gluten.metrics.MetricsUpdater +import org.apache.gluten.substrait.`type`.TypeBuilder +import org.apache.gluten.substrait.SubstraitContext +import org.apache.gluten.substrait.extensions.ExtensionBuilder +import org.apache.gluten.substrait.rel.{RelBuilder, RelNode} + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, LessThan, Literal, Rand} +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.types.DoubleType + +import scala.collection.JavaConverters._ + +/** + * SampleExec supports two sampling methods: with replacement and without replacement. This + * transformer currently supports only sampling without replacement. For sampling without + * replacement, sampleExec uses `seed + partitionId` as the seed for each partition. The `upperBound + * \- lowerBound` value is used as the fraction, and the XORShiftRandom number generator is + * employed. Each row undergoes a Bernoulli trial, and if the generated random number falls within + * the range [lowerBound, upperBound), the row is included; otherwise, it is skipped. + * + * This transformer converts sampleExec to a Substrait Filter relation, achieving a similar sampling + * effect through the filter op with rand sampling expression. Specifically, the `upperBound - + * lowerBound` value is used as the fraction, and the node be translated to `filter(rand(seed + + * partitionId) < fraction)` for random sampling. + */ +case class SampleExecTransformer( + lowerBound: Double, + upperBound: Double, + withReplacement: Boolean, + seed: Long, + child: SparkPlan) + extends UnaryTransformSupport + with Logging { + def fraction: Double = upperBound - lowerBound + + def condition: Expression = { + val randExpr: Expression = Rand(seed) + val sampleRateExpr: Expression = Literal(fraction, DoubleType) + LessThan(randExpr, sampleRateExpr) + } + + override def output: Seq[Attribute] = child.output + + // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. + @transient override lazy val metrics = + BackendsApiManager.getMetricsApiInstance.genSampleTransformerMetrics(sparkContext) + + override def metricsUpdater(): MetricsUpdater = + BackendsApiManager.getMetricsApiInstance.genSampleTransformerMetricsUpdater(metrics) + + def getRelNode( + context: SubstraitContext, + condExpr: Expression, + originalInputAttributes: Seq[Attribute], + operatorId: Long, + input: RelNode, + validation: Boolean): RelNode = { + assert(condExpr != null) + val args = context.registeredFunction + val condExprNode = ExpressionConverter + .replaceWithExpressionTransformer(condExpr, attributeSeq = originalInputAttributes) + .doTransform(args) + + if (!validation) { + RelBuilder.makeFilterRel(input, condExprNode, context, operatorId) + } else { + // Use a extension node to send the input types through Substrait plan for validation. + val inputTypeNodeList = originalInputAttributes + .map(attr => ConverterUtils.getTypeNode(attr.dataType, attr.nullable)) + .asJava + val extensionNode = ExtensionBuilder.makeAdvancedExtension( + BackendsApiManager.getTransformerApiInstance.packPBMessage( + TypeBuilder.makeStruct(false, inputTypeNodeList).toProtobuf)) + RelBuilder.makeFilterRel(input, condExprNode, extensionNode, context, operatorId) + } + } + + override protected def doValidateInternal(): ValidationResult = { + if (withReplacement) { + return ValidationResult.notOk( + "Unsupported sample exec in native with " + + s"withReplacement parameter is $withReplacement") + } + val substraitContext = new SubstraitContext + val operatorId = substraitContext.nextOperatorId((this.nodeName)) + // Firstly, need to check if the Substrait plan for this operator can be successfully generated. + val relNode = + getRelNode(substraitContext, condition, child.output, operatorId, null, validation = true) + // Then, validate the generated plan in native engine. + doNativeValidation(substraitContext, relNode) + } + + override def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + val operatorId = context.nextOperatorId(this.nodeName) + val currRel = + getRelNode(context, condition, child.output, operatorId, childCtx.root, validation = false) + assert(currRel != null, "Filter rel should be valid.") + TransformContext(childCtx.outputAttributes, output, currRel) + } + + override protected def withNewChildInternal(newChild: SparkPlan): SampleExecTransformer = + copy(child = newChild) +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 84a2ec5c6ec8..e0aa0c26bb35 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -432,6 +432,15 @@ object OffloadOthers { child, plan.evalType) } + case plan: SampleExec => + logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") + val child = plan.child + BackendsApiManager.getSparkPlanExecApiInstance.genSampleExecTransformer( + plan.lowerBound, + plan.upperBound, + plan.withReplacement, + plan.seed, + child) case p if !p.isInstanceOf[GlutenPlan] => logDebug(s"Transformation for ${p.getClass} is currently not supported.") val children = plan.children diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index c9fcc52aa091..7ce9ffc52d67 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -500,6 +500,15 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { plan.child, offset) transformer.doValidate().tagOnFallback(plan) + case plan: SampleExec => + val transformer = BackendsApiManager.getSparkPlanExecApiInstance.genSampleExecTransformer( + plan.lowerBound, + plan.upperBound, + plan.withReplacement, + plan.seed, + plan.child + ) + transformer.doValidate().tagOnFallback(plan) case _ => // Currently we assume a plan to be transformable by default. } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index d4bd9926a84c..56b63ef8457a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -192,6 +192,9 @@ object Validators { case p if HiveTableScanExecTransformer.isHiveTableScan(p) && !conf.enableColumnarHiveTableScan => fail(p) + case p: SampleExec + if !(conf.enableColumnarSample && BackendsApiManager.getSettings.supportSampleExec()) => + fail(p) case _ => pass() } } diff --git a/gluten-data/src/main/scala/org/apache/gluten/metrics/SampleMetricsUpdater.scala b/gluten-data/src/main/scala/org/apache/gluten/metrics/SampleMetricsUpdater.scala new file mode 100644 index 000000000000..a108a5b7979d --- /dev/null +++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/SampleMetricsUpdater.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.metrics + +import org.apache.spark.sql.execution.metric.SQLMetric + +class SampleMetricsUpdater(val metrics: Map[String, SQLMetric]) extends MetricsUpdater { + + override def updateNativeMetrics(opMetrics: IOperatorMetrics): Unit = { + if (opMetrics != null) { + val operatorMetrics = opMetrics.asInstanceOf[OperatorMetrics] + metrics("numOutputRows") += operatorMetrics.outputRows + metrics("outputVectors") += operatorMetrics.outputVectors + metrics("outputBytes") += operatorMetrics.outputBytes + metrics("cpuCount") += operatorMetrics.cpuCount + metrics("wallNanos") += operatorMetrics.wallNanos + metrics("peakMemoryBytes") += operatorMetrics.peakMemoryBytes + metrics("numMemoryAllocations") += operatorMetrics.numMemoryAllocations + } + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index d353c75c3a51..c9a62b8b748d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -85,6 +85,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { def enableColumnarBroadcastJoin: Boolean = conf.getConf(COLUMNAR_BROADCAST_JOIN_ENABLED) + def enableColumnarSample: Boolean = conf.getConf(COLUMNAR_SAMPLE_ENABLED) + def enableColumnarArrowUDF: Boolean = conf.getConf(COLUMNAR_ARROW_UDF_ENABLED) def enableColumnarCoalesce: Boolean = conf.getConf(COLUMNAR_COALESCE_ENABLED) @@ -1772,6 +1774,13 @@ object GlutenConfig { .booleanConf .createWithDefault(true) + val COLUMNAR_SAMPLE_ENABLED = + buildConf("spark.gluten.sql.columnarSampleEnabled") + .internal() + .doc("Disable or enable columnar sample.") + .booleanConf + .createWithDefault(false) + val CACHE_WHOLE_STAGE_TRANSFORMER_CONTEXT = buildConf("spark.gluten.sql.cacheWholeStageTransformerContext") .internal() From 8f044051674e2ecb2fa378b3f01e9bf121cfad4f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 28 May 2024 15:01:46 +0800 Subject: [PATCH 160/402] [VL] Include ClickBench benchmark in gluten-it (#5887) --- tools/gluten-it/README.md | 2 +- .../{tpc/TpcMixin.java => BaseMixin.java} | 23 ++- .../integration/{tpc/Tpc.java => Cli.java} | 20 +-- .../{tpc => }/command/DataGenMixin.java | 10 +- .../{tpc => }/command/DataGenOnly.java | 6 +- .../{tpc => }/command/Parameterized.java | 18 ++- .../{tpc => }/command/Queries.java | 10 +- .../{tpc => }/command/QueriesCompare.java | 11 +- .../{tpc => }/command/QueriesMixin.java | 10 +- .../{tpc => }/command/SparkRunModes.java | 2 +- .../{tpc => }/command/SparkShell.java | 11 +- .../main/resources/clickbench-queries/q1.sql | 1 + .../main/resources/clickbench-queries/q10.sql | 1 + .../main/resources/clickbench-queries/q11.sql | 1 + .../main/resources/clickbench-queries/q12.sql | 1 + .../main/resources/clickbench-queries/q13.sql | 1 + .../main/resources/clickbench-queries/q14.sql | 1 + .../main/resources/clickbench-queries/q15.sql | 1 + .../main/resources/clickbench-queries/q16.sql | 1 + .../main/resources/clickbench-queries/q17.sql | 1 + .../main/resources/clickbench-queries/q18.sql | 1 + .../main/resources/clickbench-queries/q19.sql | 1 + .../main/resources/clickbench-queries/q2.sql | 1 + .../main/resources/clickbench-queries/q20.sql | 1 + .../main/resources/clickbench-queries/q21.sql | 1 + .../main/resources/clickbench-queries/q22.sql | 1 + .../main/resources/clickbench-queries/q23.sql | 1 + .../main/resources/clickbench-queries/q24.sql | 1 + .../main/resources/clickbench-queries/q25.sql | 1 + .../main/resources/clickbench-queries/q26.sql | 1 + .../main/resources/clickbench-queries/q27.sql | 1 + .../main/resources/clickbench-queries/q28.sql | 1 + .../main/resources/clickbench-queries/q29.sql | 1 + .../main/resources/clickbench-queries/q3.sql | 1 + .../main/resources/clickbench-queries/q30.sql | 1 + .../main/resources/clickbench-queries/q31.sql | 1 + .../main/resources/clickbench-queries/q32.sql | 1 + .../main/resources/clickbench-queries/q33.sql | 1 + .../main/resources/clickbench-queries/q34.sql | 1 + .../main/resources/clickbench-queries/q35.sql | 1 + .../main/resources/clickbench-queries/q36.sql | 1 + .../main/resources/clickbench-queries/q37.sql | 1 + .../main/resources/clickbench-queries/q38.sql | 1 + .../main/resources/clickbench-queries/q39.sql | 1 + .../main/resources/clickbench-queries/q4.sql | 1 + .../main/resources/clickbench-queries/q40.sql | 1 + .../main/resources/clickbench-queries/q41.sql | 1 + .../main/resources/clickbench-queries/q42.sql | 1 + .../main/resources/clickbench-queries/q43.sql | 1 + .../main/resources/clickbench-queries/q5.sql | 1 + .../main/resources/clickbench-queries/q6.sql | 1 + .../main/resources/clickbench-queries/q7.sql | 1 + .../main/resources/clickbench-queries/q8.sql | 1 + .../main/resources/clickbench-queries/q9.sql | 1 + .../integration/{tpc => }/Constants.scala | 2 +- .../integration/{tpc => }/DataGen.scala | 2 +- .../TpcRunner.scala => QueryRunner.scala} | 39 +---- .../integration/{tpc => }/ShimUtils.scala | 2 +- .../{tpc/TpcSuite.scala => Suite.scala} | 32 ++-- .../gluten/integration/TableCreator.scala | 50 ++++++ .../{tpc => }/action/Actions.scala | 8 +- .../{tpc => }/action/DataGenOnly.scala | 10 +- .../{tpc => }/action/Parameterized.scala | 48 +++--- .../{tpc => }/action/Queries.scala | 32 ++-- .../{tpc => }/action/QueriesCompare.scala | 36 +++-- .../{tpc => }/action/SparkShell.scala | 20 +-- .../{tpc => }/action/TableFormatter.scala | 2 +- .../clickbench/ClickBenchDataGen.scala | 45 ++++++ .../clickbench/ClickBenchSuite.scala | 99 ++++++++++++ .../clickbench/ClickBenchTableCreator.scala | 150 ++++++++++++++++++ .../{tpc => }/ds/TpcdsDataGen.scala | 4 +- .../integration/{tpc => }/ds/TpcdsSuite.scala | 35 ++-- .../integration/{tpc => }/h/TpchDataGen.scala | 4 +- .../integration/{tpc => }/h/TpchSuite.scala | 35 ++-- ...eryRunner.scala => SparkQueryRunner.scala} | 6 +- .../{tpc/TpcTest.java => CliTest.java} | 6 +- tools/gluten-it/sbin/gluten-it.sh | 2 +- 77 files changed, 617 insertions(+), 218 deletions(-) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc/TpcMixin.java => BaseMixin.java} (89%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc/Tpc.java => Cli.java} (68%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/DataGenMixin.java (83%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/DataGenOnly.java (90%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/Parameterized.java (92%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/Queries.java (83%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/QueriesCompare.java (81%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/QueriesMixin.java (95%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/SparkRunModes.java (99%) rename tools/gluten-it/common/src/main/java/org/apache/gluten/integration/{tpc => }/command/SparkShell.java (80%) create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q1.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q10.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q11.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q12.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q13.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q14.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q15.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q16.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q17.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q18.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q19.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q2.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q20.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q21.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q22.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q23.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q24.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q25.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q26.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q27.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q28.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q29.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q3.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q30.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q31.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q32.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q33.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q34.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q35.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q36.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q37.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q38.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q39.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q4.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q40.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q41.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q42.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q43.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q5.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q6.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q7.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q8.sql create mode 100644 tools/gluten-it/common/src/main/resources/clickbench-queries/q9.sql rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/Constants.scala (99%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/DataGen.scala (98%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc/TpcRunner.scala => QueryRunner.scala} (54%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/ShimUtils.scala (97%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc/TpcSuite.scala => Suite.scala} (89%) create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/TableCreator.scala rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/Actions.scala (82%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/DataGenOnly.scala (76%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/Parameterized.scala (88%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/Queries.scala (87%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/QueriesCompare.scala (89%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/SparkShell.scala (58%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/action/TableFormatter.scala (98%) create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchTableCreator.scala rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/ds/TpcdsDataGen.scala (99%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/ds/TpcdsSuite.scala (78%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/h/TpchDataGen.scala (98%) rename tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/{tpc => }/h/TpchSuite.scala (71%) rename tools/gluten-it/common/src/main/scala/org/apache/spark/sql/{QueryRunner.scala => SparkQueryRunner.scala} (98%) rename tools/gluten-it/common/src/test/java/org/apache/gluten/integration/{tpc/TpcTest.java => CliTest.java} (90%) diff --git a/tools/gluten-it/README.md b/tools/gluten-it/README.md index 59ae55e14f18..37ed7e82b4f3 100644 --- a/tools/gluten-it/README.md +++ b/tools/gluten-it/README.md @@ -30,7 +30,7 @@ sbin/gluten-it.sh ``` Usage: gluten-it [-hV] [COMMAND] -Gluten integration test using TPC benchmark's data and queries. +Gluten integration test using various of benchmark's data and queries. -h, --help Show this help message and exit. -V, --version Print version information and exit. Commands: diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/TpcMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java similarity index 89% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/TpcMixin.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java index c0313fe77b10..41d244871b75 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/TpcMixin.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java @@ -14,12 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc; +package org.apache.gluten.integration; -import org.apache.gluten.integration.tpc.action.Action; -import org.apache.gluten.integration.tpc.command.SparkRunModes; -import org.apache.gluten.integration.tpc.ds.TpcdsSuite; -import org.apache.gluten.integration.tpc.h.TpchSuite; +import org.apache.gluten.integration.action.Action; +import org.apache.gluten.integration.clickbench.ClickBenchSuite; +import org.apache.gluten.integration.command.SparkRunModes; +import org.apache.gluten.integration.ds.TpcdsSuite; +import org.apache.gluten.integration.h.TpchSuite; import org.apache.log4j.Level; import org.apache.spark.SparkConf; import picocli.CommandLine; @@ -30,9 +31,9 @@ import java.util.HashMap; import java.util.Map; -public class TpcMixin { +public class BaseMixin { - @CommandLine.Option(required = true, names = {"--benchmark-type"}, description = "TPC benchmark type: h, ds", defaultValue = "h") + @CommandLine.Option(required = true, names = {"--benchmark-type"}, description = "Benchmark type: h, ds, clickbench", defaultValue = "h") private String benchmarkType; @CommandLine.Option(names = {"-p", "--preset"}, description = "Preset used: vanilla, velox, velox-with-celeborn, velox-with-uniffle...", defaultValue = "velox") @@ -124,7 +125,7 @@ public Integer runActions(Action[] actions) { mergeMapSafe(extraSparkConf, runModeEnumeration.extraSparkConf())).asScala().toMap( Predef.conforms()); - final TpcSuite suite; + final Suite suite; switch (benchmarkType) { case "h": suite = new TpchSuite(runModeEnumeration.getSparkMasterUrl(), actions, testConf, @@ -138,6 +139,12 @@ public Integer runActions(Action[] actions) { enableHsUi, hsUiPort, offHeapSize, disableAqe, disableBhj, disableWscg, shufflePartitions, minimumScanPartitions); break; + case "clickbench": + suite = new ClickBenchSuite(runModeEnumeration.getSparkMasterUrl(), actions, testConf, + baselineConf, extraSparkConfScala, level, errorOnMemLeak, enableUi, + enableHsUi, hsUiPort, offHeapSize, disableAqe, disableBhj, + disableWscg, shufflePartitions, minimumScanPartitions); + break; default: throw new IllegalArgumentException("TPC benchmark type not found: " + benchmarkType); } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/Tpc.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/Cli.java similarity index 68% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/Tpc.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/Cli.java index ceaf71b54672..be0bf57220fe 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/Tpc.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/Cli.java @@ -14,26 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc; +package org.apache.gluten.integration; -import org.apache.gluten.integration.tpc.command.DataGenOnly; -import org.apache.gluten.integration.tpc.command.Parameterized; -import org.apache.gluten.integration.tpc.command.Queries; -import org.apache.gluten.integration.tpc.command.QueriesCompare; -import org.apache.gluten.integration.tpc.command.SparkShell; +import org.apache.gluten.integration.command.DataGenOnly; +import org.apache.gluten.integration.command.Parameterized; +import org.apache.gluten.integration.command.Queries; +import org.apache.gluten.integration.command.QueriesCompare; +import org.apache.gluten.integration.command.SparkShell; import picocli.CommandLine; @CommandLine.Command(name = "gluten-it", mixinStandardHelpOptions = true, showDefaultValues = true, subcommands = {DataGenOnly.class, Queries.class, QueriesCompare.class, SparkShell.class, Parameterized.class}, - description = "Gluten integration test using TPC benchmark's data and queries.") -public class Tpc { + description = "Gluten integration test using various of benchmark's data and queries.") +public class Cli { - private Tpc() { + private Cli() { } public static void main(String... args) { - final CommandLine cmd = new CommandLine(new Tpc()); + final CommandLine cmd = new CommandLine(new Cli()); final int exitCode = cmd.execute(args); System.exit(exitCode); } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/DataGenMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/DataGenMixin.java similarity index 83% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/DataGenMixin.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/DataGenMixin.java index 72ca0c699673..0682f5601a92 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/DataGenMixin.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/DataGenMixin.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; -import org.apache.gluten.integration.tpc.action.Action; +import org.apache.gluten.integration.action.Action; import picocli.CommandLine; public class DataGenMixin { @@ -33,10 +33,14 @@ public Action[] makeActions() { if (skipDataGen) { return new Action[0]; } - return new Action[]{new org.apache.gluten.integration.tpc.action.DataGenOnly(scale, genPartitionedData)}; + return new Action[]{new org.apache.gluten.integration.action.DataGenOnly(scale, genPartitionedData)}; } public double getScale() { return scale; } + + public boolean genPartitionedData() { + return genPartitionedData; + } } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/DataGenOnly.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/DataGenOnly.java similarity index 90% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/DataGenOnly.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/DataGenOnly.java index 11e14426fab0..f1ac4888885b 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/DataGenOnly.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/DataGenOnly.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; -import org.apache.gluten.integration.tpc.TpcMixin; +import org.apache.gluten.integration.BaseMixin; import picocli.CommandLine; import java.util.concurrent.Callable; @@ -26,7 +26,7 @@ description = "Generate data only.") public class DataGenOnly implements Callable { @CommandLine.Mixin - private TpcMixin mixin; + private BaseMixin mixin; @CommandLine.Mixin private DataGenMixin dataGenMixin; diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/Parameterized.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java similarity index 92% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/Parameterized.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java index bf7d89fe60ca..7e1234e7665d 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/Parameterized.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; import com.google.common.base.Preconditions; -import org.apache.gluten.integration.tpc.TpcMixin; -import org.apache.gluten.integration.tpc.action.Dim; -import org.apache.gluten.integration.tpc.action.DimKv; -import org.apache.gluten.integration.tpc.action.DimValue; +import org.apache.gluten.integration.BaseMixin; +import org.apache.gluten.integration.action.Dim; +import org.apache.gluten.integration.action.DimKv; +import org.apache.gluten.integration.action.DimValue; import org.apache.commons.lang3.ArrayUtils; import picocli.CommandLine; import scala.Tuple2; @@ -38,7 +38,7 @@ description = "Run queries with parameterized configurations") public class Parameterized implements Callable { @CommandLine.Mixin - private TpcMixin mixin; + private BaseMixin mixin; @CommandLine.Mixin private DataGenMixin dataGenMixin; @@ -62,6 +62,7 @@ public class Parameterized implements Callable { private static final Pattern dimPattern2 = Pattern.compile("([^,:]+)((?:,[^=,]+=[^=,]+)+)"); private static final Pattern excludedDimsPattern = Pattern.compile("[\\w-]+:[^,:]+(?:,[\\w-]+:[^,:]+)*"); + @Override public Integer call() throws Exception { final Map>>> parsed = new HashMap<>(); @@ -129,8 +130,9 @@ public Integer call() throws Exception { .collect(Collectors.toList())).asScala())).collect(Collectors.toList())).asScala() )).collect(Collectors.toList())).asScala(); - org.apache.gluten.integration.tpc.action.Parameterized parameterized = - new org.apache.gluten.integration.tpc.action.Parameterized(dataGenMixin.getScale(), queriesMixin.queries(), + org.apache.gluten.integration.action.Parameterized parameterized = + new org.apache.gluten.integration.action.Parameterized(dataGenMixin.getScale(), + dataGenMixin.genPartitionedData(), queriesMixin.queries(), queriesMixin.explain(), queriesMixin.iterations(), warmupIterations, parsedDims, excludedCombinations, metrics); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), parameterized)); diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/Queries.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java similarity index 83% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/Queries.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java index 53d46cc5eac3..f0c07b41538b 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/Queries.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; -import org.apache.gluten.integration.tpc.TpcMixin; +import org.apache.gluten.integration.BaseMixin; import org.apache.commons.lang3.ArrayUtils; import picocli.CommandLine; @@ -27,7 +27,7 @@ description = "Run queries.") public class Queries implements Callable { @CommandLine.Mixin - private TpcMixin mixin; + private BaseMixin mixin; @CommandLine.Mixin private DataGenMixin dataGenMixin; @@ -40,8 +40,8 @@ public class Queries implements Callable { @Override public Integer call() throws Exception { - org.apache.gluten.integration.tpc.action.Queries queries = - new org.apache.gluten.integration.tpc.action.Queries(dataGenMixin.getScale(), queriesMixin.queries(), + org.apache.gluten.integration.action.Queries queries = + new org.apache.gluten.integration.action.Queries(dataGenMixin.getScale(), dataGenMixin.genPartitionedData(), queriesMixin.queries(), queriesMixin.explain(), queriesMixin.iterations(), randomKillTasks); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), queries)); } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/QueriesCompare.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java similarity index 81% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/QueriesCompare.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java index d4c0c684dd10..42b00f94cece 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/QueriesCompare.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; -import org.apache.gluten.integration.tpc.TpcMixin; +import org.apache.gluten.integration.BaseMixin; import org.apache.commons.lang3.ArrayUtils; import picocli.CommandLine; @@ -27,7 +27,7 @@ description = "Run queries and do result comparison with baseline preset.") public class QueriesCompare implements Callable { @CommandLine.Mixin - private TpcMixin mixin; + private BaseMixin mixin; @CommandLine.Mixin private DataGenMixin dataGenMixin; @@ -37,8 +37,9 @@ public class QueriesCompare implements Callable { @Override public Integer call() throws Exception { - org.apache.gluten.integration.tpc.action.QueriesCompare queriesCompare = - new org.apache.gluten.integration.tpc.action.QueriesCompare(dataGenMixin.getScale(), queriesMixin.queries(), + org.apache.gluten.integration.action.QueriesCompare queriesCompare = + new org.apache.gluten.integration.action.QueriesCompare(dataGenMixin.getScale(), + dataGenMixin.genPartitionedData(), queriesMixin.queries(), queriesMixin.explain(), queriesMixin.iterations()); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), queriesCompare)); } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/QueriesMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java similarity index 95% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/QueriesMixin.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java index f514883640b7..fc93f968c85c 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/QueriesMixin.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; import com.google.common.base.Preconditions; -import org.apache.gluten.integration.tpc.TpcSuite; -import org.apache.gluten.integration.tpc.action.Actions; +import org.apache.gluten.integration.Suite; +import org.apache.gluten.integration.action.Actions; import picocli.CommandLine; import scala.collection.Seq; import scala.collection.JavaConverters; @@ -53,7 +53,7 @@ public int iterations() { public Actions.QuerySelector queries() { return new Actions.QuerySelector() { @Override - public Seq select(TpcSuite suite) { + public Seq select(Suite suite) { final List all = select0(suite); final Division div = Division.parse(shard); final List out = div(all, div); @@ -81,7 +81,7 @@ private List div(List from, Division div) { return out; } - private List select0(TpcSuite suite) { + private List select0(Suite suite) { final String[] queryIds = queries; final String[] excludedQueryIds = excludedQueries; if (queryIds.length > 0 && excludedQueryIds.length > 0) { diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/SparkRunModes.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java similarity index 99% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/SparkRunModes.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java index 987099902b03..f5a5c73a682f 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/SparkRunModes.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; import org.apache.spark.launcher.SparkLauncher; import org.apache.spark.util.Utils; diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/SparkShell.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkShell.java similarity index 80% rename from tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/SparkShell.java rename to tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkShell.java index f16cedb86cb4..9c2829e92313 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/tpc/command/SparkShell.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkShell.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.command; +package org.apache.gluten.integration.command; -import org.apache.gluten.integration.tpc.TpcMixin; +import org.apache.gluten.integration.BaseMixin; import org.apache.commons.lang3.ArrayUtils; import picocli.CommandLine; @@ -27,15 +27,16 @@ description = "Open a standard Spark shell.") public class SparkShell implements Callable { @CommandLine.Mixin - private TpcMixin mixin; + private BaseMixin mixin; @CommandLine.Mixin private DataGenMixin dataGenMixin; @Override public Integer call() throws Exception { - org.apache.gluten.integration.tpc.action.SparkShell sparkShell = - new org.apache.gluten.integration.tpc.action.SparkShell(dataGenMixin.getScale()); + org.apache.gluten.integration.action.SparkShell sparkShell = + new org.apache.gluten.integration.action.SparkShell(dataGenMixin.getScale(), + dataGenMixin.genPartitionedData()); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), sparkShell)); } } diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q1.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q1.sql new file mode 100644 index 000000000000..c70aa7a844d7 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q1.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q10.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q10.sql new file mode 100644 index 000000000000..f4a9ee3446e8 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q10.sql @@ -0,0 +1 @@ +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q11.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q11.sql new file mode 100644 index 000000000000..d0ea7e3b386a --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q11.sql @@ -0,0 +1 @@ +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q12.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q12.sql new file mode 100644 index 000000000000..2a316d173429 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q12.sql @@ -0,0 +1 @@ +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q13.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q13.sql new file mode 100644 index 000000000000..7a0254690e79 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q13.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q14.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q14.sql new file mode 100644 index 000000000000..4ce0feed26b9 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q14.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q15.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q15.sql new file mode 100644 index 000000000000..8e85255a0a2d --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q15.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q16.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q16.sql new file mode 100644 index 000000000000..f959e98a6f5b --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q16.sql @@ -0,0 +1 @@ +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q17.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q17.sql new file mode 100644 index 000000000000..50b1f38320a9 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q17.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q18.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q18.sql new file mode 100644 index 000000000000..454cdb507fb4 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q18.sql @@ -0,0 +1 @@ +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q19.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q19.sql new file mode 100644 index 000000000000..fbd104900498 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q19.sql @@ -0,0 +1 @@ +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, extract(minute FROM EventTime), SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q2.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q2.sql new file mode 100644 index 000000000000..ad8031a76cf7 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q2.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q20.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q20.sql new file mode 100644 index 000000000000..3f84066e6a11 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q20.sql @@ -0,0 +1 @@ +SELECT UserID FROM hits WHERE UserID = 435090932899640449; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q21.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q21.sql new file mode 100644 index 000000000000..4426afa35fc3 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q21.sql @@ -0,0 +1 @@ +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q22.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q22.sql new file mode 100644 index 000000000000..bd28609bd201 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q22.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q23.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q23.sql new file mode 100644 index 000000000000..4ee87ac4528d --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q23.sql @@ -0,0 +1 @@ +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q24.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q24.sql new file mode 100644 index 000000000000..935169e37996 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q24.sql @@ -0,0 +1 @@ +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q25.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q25.sql new file mode 100644 index 000000000000..1bcfd4c1e4ba --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q25.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q26.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q26.sql new file mode 100644 index 000000000000..58ea7610cdcd --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q26.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q27.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q27.sql new file mode 100644 index 000000000000..88ed7ba3e156 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q27.sql @@ -0,0 +1 @@ +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q28.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q28.sql new file mode 100644 index 000000000000..d18ffcd72cd3 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q28.sql @@ -0,0 +1 @@ +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q29.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q29.sql new file mode 100644 index 000000000000..86d6f204be26 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q29.sql @@ -0,0 +1 @@ +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q3.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q3.sql new file mode 100644 index 000000000000..7db4dc2fe986 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q3.sql @@ -0,0 +1 @@ +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q30.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q30.sql new file mode 100644 index 000000000000..630450b43269 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q30.sql @@ -0,0 +1 @@ +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q31.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q31.sql new file mode 100644 index 000000000000..f8a80d9c6fb9 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q31.sql @@ -0,0 +1 @@ +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q32.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q32.sql new file mode 100644 index 000000000000..ba8ac2f91d27 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q32.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q33.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q33.sql new file mode 100644 index 000000000000..893773a20001 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q33.sql @@ -0,0 +1 @@ +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q34.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q34.sql new file mode 100644 index 000000000000..f00a3e8efb93 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q34.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q35.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q35.sql new file mode 100644 index 000000000000..213753083047 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q35.sql @@ -0,0 +1 @@ +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q36.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q36.sql new file mode 100644 index 000000000000..581e5e389fcd --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q36.sql @@ -0,0 +1 @@ +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q37.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q37.sql new file mode 100644 index 000000000000..7aa52984543b --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q37.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-01' AND EventDate <= DATE '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q38.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q38.sql new file mode 100644 index 000000000000..b1a580dba070 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q38.sql @@ -0,0 +1 @@ +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-01' AND EventDate <= DATE '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q39.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q39.sql new file mode 100644 index 000000000000..93b691f37cd2 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q39.sql @@ -0,0 +1 @@ +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-01' AND EventDate <= DATE '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC OFFSET 1000 LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q4.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q4.sql new file mode 100644 index 000000000000..e953498fe3ab --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q4.sql @@ -0,0 +1 @@ +SELECT AVG(UserID) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q40.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q40.sql new file mode 100644 index 000000000000..d97b60772000 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q40.sql @@ -0,0 +1 @@ +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-01' AND EventDate <= DATE '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END, URL ORDER BY PageViews DESC OFFSET 1000 LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q41.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q41.sql new file mode 100644 index 000000000000..321a06e3e701 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q41.sql @@ -0,0 +1 @@ +SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-01' AND EventDate <= DATE '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC OFFSET 100 LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q42.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q42.sql new file mode 100644 index 000000000000..46b81c5be467 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q42.sql @@ -0,0 +1 @@ +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-01' AND EventDate <= DATE '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC OFFSET 10000 LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q43.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q43.sql new file mode 100644 index 000000000000..b54c0921149b --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q43.sql @@ -0,0 +1 @@ +SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= DATE '2013-07-14' AND EventDate <= DATE '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) OFFSET 1000 LIMIT 10; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q5.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q5.sql new file mode 100644 index 000000000000..2f9baf82535a --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q5.sql @@ -0,0 +1 @@ +SELECT COUNT(DISTINCT UserID) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q6.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q6.sql new file mode 100644 index 000000000000..e9615f8fd345 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q6.sql @@ -0,0 +1 @@ +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q7.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q7.sql new file mode 100644 index 000000000000..03fbb82d68b8 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q7.sql @@ -0,0 +1 @@ +SELECT MIN(EventDate), MAX(EventDate) FROM hits; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q8.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q8.sql new file mode 100644 index 000000000000..25a10b112d56 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q8.sql @@ -0,0 +1 @@ +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; diff --git a/tools/gluten-it/common/src/main/resources/clickbench-queries/q9.sql b/tools/gluten-it/common/src/main/resources/clickbench-queries/q9.sql new file mode 100644 index 000000000000..a6bf47e4fea1 --- /dev/null +++ b/tools/gluten-it/common/src/main/resources/clickbench-queries/q9.sql @@ -0,0 +1 @@ +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala similarity index 99% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala index d39a16c325ef..50766f3a91d1 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/Constants.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc +package org.apache.gluten.integration import org.apache.spark.SparkConf import org.apache.spark.sql.TypeUtils diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala similarity index 98% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala index e810a4dc2316..6b2d4ec71b86 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/DataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/DataGen.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc +package org.apache.gluten.integration import org.apache.spark.sql.types.{DataType, StructField, StructType} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala similarity index 54% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala index 908b8206eecd..88e8e2250fd8 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala @@ -14,26 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc - -import org.apache.spark.sql.{AnalysisException, QueryRunner, RunResult, SparkSession} +package org.apache.gluten.integration import com.google.common.base.Preconditions -import org.apache.commons.io.FileUtils +import org.apache.spark.sql.{RunResult, SparkQueryRunner, SparkSession} import java.io.File -class TpcRunner(val queryResourceFolder: String, val dataPath: String) { +class QueryRunner(val queryResourceFolder: String, val dataPath: String) { Preconditions.checkState( new File(dataPath).exists(), s"Data not found at $dataPath, try using command ` data-gen-only ` to generate it first.", Array(): _*) - def createTables(spark: SparkSession): Unit = { - TpcRunner.createTables(spark, dataPath) + def createTables(creator: TableCreator, spark: SparkSession): Unit = { + creator.create(spark, dataPath) } - def runTpcQuery( + def runQuery( spark: SparkSession, desc: String, caseId: String, @@ -41,29 +39,8 @@ class TpcRunner(val queryResourceFolder: String, val dataPath: String) { metrics: Array[String] = Array(), randomKillTasks: Boolean = false): RunResult = { val path = "%s/%s.sql".format(queryResourceFolder, caseId) - QueryRunner.runTpcQuery(spark, desc, path, explain, metrics, randomKillTasks) + SparkQueryRunner.runQuery(spark, desc, path, explain, metrics, randomKillTasks) } } -object TpcRunner { - def createTables(spark: SparkSession, dataPath: String): Unit = { - val files = new File(dataPath).listFiles() - files.foreach(file => { - if (spark.catalog.tableExists(file.getName)) { - println("Table exists: " + file.getName) - } else { - println("Creating catalog table: " + file.getName) - spark.catalog.createTable(file.getName, file.getAbsolutePath, "parquet") - try { - spark.catalog.recoverPartitions(file.getName) - } catch { - case _: AnalysisException => - } - } - }) - } - - private def delete(path: String): Unit = { - FileUtils.forceDelete(new File(path)) - } -} +object QueryRunner {} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ShimUtils.scala similarity index 97% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ShimUtils.scala index 19e15df5cca7..d2986bfa7c0c 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ShimUtils.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ShimUtils.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.gluten.integration.tpc +package org.apache.gluten.integration import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala similarity index 89% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala index f7605e273eb1..9e31e11713a5 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/TpcSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala @@ -14,22 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc - -import org.apache.gluten.integration.tpc.action.Action +package org.apache.gluten.integration +import org.apache.gluten.integration.action.Action +import org.apache.log4j.{Level, LogManager} import org.apache.spark.SparkConf import org.apache.spark.deploy.history.HistoryServerHelper import org.apache.spark.network.util.ByteUnit import org.apache.spark.sql.ConfUtils.ConfImplicits._ import org.apache.spark.sql.SparkSessionSwitcher -import org.apache.log4j.{Level, LogManager} - import java.io.File import java.util.Scanner -abstract class TpcSuite( +abstract class Suite( private val masterUrl: String, private val actions: Array[Action], private val testConf: SparkConf, @@ -49,7 +47,7 @@ abstract class TpcSuite( resetLogLevel() - private[tpc] val sessionSwitcher: SparkSessionSwitcher = + private[integration] val sessionSwitcher: SparkSessionSwitcher = new SparkSessionSwitcher(masterUrl, logLevel.toString) // define initial configs @@ -153,32 +151,32 @@ abstract class TpcSuite( } } + def tableCreator(): TableCreator + private def resetLogLevel(): Unit = { LogManager.getRootLogger.setLevel(logLevel) } - private[tpc] def getBaselineConf(): SparkConf = { + private[integration] def getBaselineConf(): SparkConf = { baselineConf.clone() } - private[tpc] def getTestConf(): SparkConf = { + private[integration] def getTestConf(): SparkConf = { testConf.clone() } protected def historyWritePath(): String - private[tpc] def dataWritePath(scale: Double): String - - private[tpc] def createDataGen(scale: Double, genPartitionedData: Boolean): DataGen + private[integration] def dataWritePath(scale: Double, genPartitionedData: Boolean): String - private[tpc] def queryResource(): String + private[integration] def createDataGen(scale: Double, genPartitionedData: Boolean): DataGen - protected def typeModifiers(): List[TypeModifier] + private[integration] def queryResource(): String - private[tpc] def allQueryIds(): Array[String] + private[integration] def allQueryIds(): Array[String] - private[tpc] def desc(): String + private[integration] def desc(): String } -object TpcSuite {} +object Suite {} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/TableCreator.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/TableCreator.scala new file mode 100644 index 000000000000..b35aceef8768 --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/TableCreator.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.integration + +import org.apache.spark.sql.{AnalysisException, SparkSession} + +import java.io.File + +trait TableCreator { + def create(spark: SparkSession, dataPath: String): Unit +} + +object TableCreator { + def discoverSchema(): TableCreator = { + DiscoverSchema + } + + private object DiscoverSchema extends TableCreator { + override def create(spark: SparkSession, dataPath: String): Unit = { + val files = new File(dataPath).listFiles() + files.foreach(file => { + if (spark.catalog.tableExists(file.getName)) { + println("Table exists: " + file.getName) + } else { + println("Creating catalog table: " + file.getName) + spark.catalog.createTable(file.getName, file.getAbsolutePath, "parquet") + try { + spark.catalog.recoverPartitions(file.getName) + } catch { + case _: AnalysisException => + } + } + }) + } + } +} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Actions.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Actions.scala similarity index 82% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Actions.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Actions.scala index 5e49b2888856..4977dda70820 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Actions.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Actions.scala @@ -14,16 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.action +package org.apache.gluten.integration.action -import org.apache.gluten.integration.tpc.TpcSuite +import org.apache.gluten.integration.Suite trait Action { - def execute(tpcSuite: TpcSuite): Boolean + def execute(suite: Suite): Boolean } object Actions { trait QuerySelector { - def select(suite: TpcSuite): Seq[String] + def select(suite: Suite): Seq[String] } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/DataGenOnly.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/DataGenOnly.scala similarity index 76% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/DataGenOnly.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/DataGenOnly.scala index 488bb19adaef..bc43834610a4 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/DataGenOnly.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/DataGenOnly.scala @@ -14,16 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.action +package org.apache.gluten.integration.action -import org.apache.gluten.integration.tpc.TpcSuite +import org.apache.gluten.integration.Suite import java.io.File case class DataGenOnly(scale: Double, genPartitionedData: Boolean) extends Action { - override def execute(tpcSuite: TpcSuite): Boolean = { - tpcSuite.sessionSwitcher.useSession("baseline", "Data Gen") - val dataGen = tpcSuite.createDataGen(scale, genPartitionedData) + override def execute(suite: Suite): Boolean = { + suite.sessionSwitcher.useSession("baseline", "Data Gen") + val dataGen = suite.createDataGen(scale, genPartitionedData) dataGen.gen() true } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala similarity index 88% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala index 6fc4e66d6f05..2871ef2de0f8 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala @@ -14,21 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.action +package org.apache.gluten.integration.action +import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.gluten.integration.action.Actions.QuerySelector import org.apache.gluten.integration.stat.RamStat -import org.apache.gluten.integration.tpc.{TpcRunner, TpcSuite} +import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} import org.apache.spark.sql.ConfUtils.ConfImplicits._ import org.apache.spark.sql.SparkSessionSwitcher -import org.apache.commons.lang3.exception.ExceptionUtils -import org.apache.gluten.integration.tpc.action.Actions.QuerySelector -import scala.collection.immutable.Map import scala.collection.mutable import scala.collection.mutable.{ArrayBuffer, ListBuffer} class Parameterized( scale: Double, + genPartitionedData: Boolean, queries: QuerySelector, explain: Boolean, iterations: Int, @@ -91,12 +91,13 @@ class Parameterized( coordinateMap } - override def execute(tpcSuite: TpcSuite): Boolean = { - val runner: TpcRunner = new TpcRunner(tpcSuite.queryResource(), tpcSuite.dataWritePath(scale)) - val allQueries = tpcSuite.allQueryIds() + override def execute(suite: Suite): Boolean = { + val runner: QueryRunner = + new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) + val allQueries = suite.allQueryIds() - val sessionSwitcher = tpcSuite.sessionSwitcher - val testConf = tpcSuite.getTestConf() + val sessionSwitcher = suite.sessionSwitcher + val testConf = suite.getTestConf() println("Prepared coordinates: ") coordinates.toList.map(_._1).zipWithIndex.foreach { @@ -112,12 +113,12 @@ class Parameterized( sessionSwitcher.registerSession(coordinate.toString, conf) } - val runQueryIds = queries.select(tpcSuite) + val runQueryIds = queries.select(suite) // warm up (0 until warmupIterations).foreach { _ => runQueryIds.foreach { queryId => - Parameterized.warmUp(queryId, tpcSuite.desc(), sessionSwitcher, runner) + Parameterized.warmUp(suite.tableCreator(), queryId, suite.desc(), sessionSwitcher, runner) } } @@ -126,12 +127,13 @@ class Parameterized( val coordinateResults = (0 until iterations).flatMap { iteration => println(s"Running tests (iteration $iteration) with coordinate $coordinate...") runQueryIds.map { queryId => - Parameterized.runTpcQuery( + Parameterized.runQuery( runner, + suite.tableCreator(), sessionSwitcher, queryId, coordinate, - tpcSuite.desc(), + suite.desc(), explain, metrics) } @@ -241,8 +243,9 @@ case class TestResultLines( } object Parameterized { - private def runTpcQuery( - runner: TpcRunner, + private def runQuery( + runner: QueryRunner, + creator: TableCreator, sessionSwitcher: SparkSessionSwitcher, id: String, coordinate: Coordinate, @@ -253,9 +256,9 @@ object Parameterized { try { val testDesc = "Gluten Spark %s %s %s".format(desc, id, coordinate) sessionSwitcher.useSession(coordinate.toString, testDesc) - runner.createTables(sessionSwitcher.spark()) + runner.createTables(creator, sessionSwitcher.spark()) val result = - runner.runTpcQuery(sessionSwitcher.spark(), testDesc, id, explain, metrics) + runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain, metrics) val resultRows = result.rows println( s"Successfully ran query $id. " + @@ -279,17 +282,18 @@ object Parameterized { } } - private[tpc] def warmUp( + private[integration] def warmUp( + creator: TableCreator, id: String, desc: String, sessionSwitcher: SparkSessionSwitcher, - runner: TpcRunner): Unit = { + runner: QueryRunner): Unit = { println(s"Warming up: Running query: $id...") try { val testDesc = "Gluten Spark %s %s warm up".format(desc, id) sessionSwitcher.useSession("test", testDesc) - runner.createTables(sessionSwitcher.spark()) - val result = runner.runTpcQuery(sessionSwitcher.spark(), testDesc, id, explain = false) + runner.createTables(creator, sessionSwitcher.spark()) + val result = runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain = false) val resultRows = result.rows println( s"Warming up: Successfully ran query $id. " + diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala similarity index 87% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala index 290b8e3f5b0c..cf24b906b2aa 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala @@ -14,32 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.action +package org.apache.gluten.integration.action -import org.apache.gluten.integration.stat.RamStat -import org.apache.gluten.integration.tpc.{TpcRunner, TpcSuite} import org.apache.commons.lang3.exception.ExceptionUtils -import org.apache.gluten.integration.tpc.action.Actions.QuerySelector +import org.apache.gluten.integration.action.Actions.QuerySelector +import org.apache.gluten.integration.stat.RamStat +import org.apache.gluten.integration.{QueryRunner, Suite} case class Queries( scale: Double, + genPartitionedData: Boolean, queries: QuerySelector, explain: Boolean, iterations: Int, randomKillTasks: Boolean) extends Action { - override def execute(tpcSuite: TpcSuite): Boolean = { - val runQueryIds = queries.select(tpcSuite) - val runner: TpcRunner = new TpcRunner(tpcSuite.queryResource(), tpcSuite.dataWritePath(scale)) + override def execute(suite: Suite): Boolean = { + val runQueryIds = queries.select(suite) + val runner: QueryRunner = + new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) val results = (0 until iterations).flatMap { iteration => println(s"Running tests (iteration $iteration)...") runQueryIds.map { queryId => - Queries.runTpcQuery( + Queries.runQuery( runner, - tpcSuite.sessionSwitcher, + suite.tableCreator(), + suite.sessionSwitcher, queryId, - tpcSuite.desc(), + suite.desc(), explain, randomKillTasks) } @@ -153,8 +156,9 @@ object Queries { None))) } - private def runTpcQuery( - runner: _root_.org.apache.gluten.integration.tpc.TpcRunner, + private def runQuery( + runner: _root_.org.apache.gluten.integration.QueryRunner, + creator: _root_.org.apache.gluten.integration.TableCreator, sessionSwitcher: _root_.org.apache.spark.sql.SparkSessionSwitcher, id: _root_.java.lang.String, desc: _root_.java.lang.String, @@ -164,8 +168,8 @@ object Queries { try { val testDesc = "Gluten Spark %s %s".format(desc, id) sessionSwitcher.useSession("test", testDesc) - runner.createTables(sessionSwitcher.spark()) - val result = runner.runTpcQuery( + runner.createTables(creator, sessionSwitcher.spark()) + val result = runner.runQuery( sessionSwitcher.spark(), testDesc, id, diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala similarity index 89% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala index 404d75cb426e..320bd61b609d 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala @@ -14,32 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.action +package org.apache.gluten.integration.action +import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.gluten.integration.action.Actions.QuerySelector import org.apache.gluten.integration.stat.RamStat -import org.apache.gluten.integration.tpc.{TpcRunner, TpcSuite} +import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} import org.apache.spark.sql.{SparkSessionSwitcher, TestUtils} -import org.apache.commons.lang3.exception.ExceptionUtils -import org.apache.gluten.integration.tpc.action.Actions.QuerySelector case class QueriesCompare( scale: Double, + genPartitionedData: Boolean, queries: QuerySelector, explain: Boolean, iterations: Int) extends Action { - override def execute(tpcSuite: TpcSuite): Boolean = { - val runner: TpcRunner = new TpcRunner(tpcSuite.queryResource(), tpcSuite.dataWritePath(scale)) - val runQueryIds = queries.select(tpcSuite) + override def execute(suite: Suite): Boolean = { + val runner: QueryRunner = + new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) + val runQueryIds = queries.select(suite) val results = (0 until iterations).flatMap { iteration => println(s"Running tests (iteration $iteration)...") runQueryIds.map { queryId => - QueriesCompare.runTpcQuery( + QueriesCompare.runQuery( + suite.tableCreator(), queryId, explain, - tpcSuite.desc(), - tpcSuite.sessionSwitcher, + suite.desc(), + suite.sessionSwitcher, runner) } }.toList @@ -179,24 +182,25 @@ object QueriesCompare { None))) } - private[tpc] def runTpcQuery( + private[integration] def runQuery( + creator: TableCreator, id: String, explain: Boolean, desc: String, sessionSwitcher: SparkSessionSwitcher, - runner: TpcRunner): TestResultLine = { + runner: QueryRunner): TestResultLine = { println(s"Running query: $id...") try { val baseLineDesc = "Vanilla Spark %s %s".format(desc, id) sessionSwitcher.useSession("baseline", baseLineDesc) - runner.createTables(sessionSwitcher.spark()) + runner.createTables(creator, sessionSwitcher.spark()) val expected = - runner.runTpcQuery(sessionSwitcher.spark(), baseLineDesc, id, explain = explain) + runner.runQuery(sessionSwitcher.spark(), baseLineDesc, id, explain = explain) val expectedRows = expected.rows val testDesc = "Gluten Spark %s %s".format(desc, id) sessionSwitcher.useSession("test", testDesc) - runner.createTables(sessionSwitcher.spark()) - val result = runner.runTpcQuery(sessionSwitcher.spark(), testDesc, id, explain = explain) + runner.createTables(creator, sessionSwitcher.spark()) + val result = runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain = explain) val resultRows = result.rows val error = TestUtils.compareAnswers(resultRows, expectedRows, sort = true) if (error.isEmpty) { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/SparkShell.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala similarity index 58% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/SparkShell.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala index 78e816955e2e..76f43cb71b35 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/SparkShell.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala @@ -14,19 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.action - -import org.apache.gluten.integration.tpc.{TpcRunner, TpcSuite} +package org.apache.gluten.integration.action +import org.apache.gluten.integration.{QueryRunner, Suite} import org.apache.spark.repl.Main -case class SparkShell(scale: Double) extends Action { - override def execute(tpcSuite: TpcSuite): Boolean = { - tpcSuite.sessionSwitcher.useSession("test", "Gluten Spark CLI") - val runner: TpcRunner = new TpcRunner(tpcSuite.queryResource(), tpcSuite.dataWritePath(scale)) - runner.createTables(tpcSuite.sessionSwitcher.spark()) - Main.sparkSession = tpcSuite.sessionSwitcher.spark() - Main.sparkContext = tpcSuite.sessionSwitcher.spark().sparkContext +case class SparkShell(scale: Double, genPartitionedData: Boolean) extends Action { + override def execute(suite: Suite): Boolean = { + suite.sessionSwitcher.useSession("test", "Gluten Spark CLI") + val runner: QueryRunner = + new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) + runner.createTables(suite.tableCreator(), suite.sessionSwitcher.spark()) + Main.sparkSession = suite.sessionSwitcher.spark() + Main.sparkContext = suite.sessionSwitcher.spark().sparkContext Main.main(Array("-usejavacp")) true } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala similarity index 98% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala index 8aeea9938e90..07e253d5e4e1 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/action/TableFormatter.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.gluten.integration.tpc.action +package org.apache.gluten.integration.action import java.io.{OutputStream, PrintStream} import scala.collection.mutable diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala new file mode 100644 index 000000000000..ba772f165190 --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.integration.clickbench + +import org.apache.commons.io.FileUtils +import org.apache.gluten.integration.DataGen +import org.apache.spark.sql.SparkSession + +import java.io.File +import scala.language.postfixOps +import scala.sys.process._ + +class ClickBenchDataGen(val spark: SparkSession, dir: String) extends DataGen { + import ClickBenchDataGen._ + override def gen(): Unit = { + println(s"Start to download ClickBench Parquet dataset from URL: $DATA_URL... ") + // Directly download from official URL. + val target = new File(dir + File.separator + FILE_NAME) + FileUtils.forceMkdirParent(target) + val code = Process(s"wget -P $dir $DATA_URL") !; + if (code != 0) { + throw new RuntimeException("Download failed") + } + println(s"ClickBench Parquet dataset successfully downloaded to $target.") + } +} + +object ClickBenchDataGen { + private val DATA_URL = "https://datasets.clickhouse.com/hits_compatible/hits.parquet" + private[clickbench] val FILE_NAME = "hits.parquet" +} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala new file mode 100644 index 000000000000..deffdb7e556a --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.integration.clickbench + +import org.apache.gluten.integration.action.Action +import org.apache.gluten.integration.{DataGen, Suite, TableCreator} +import org.apache.log4j.Level +import org.apache.spark.SparkConf + +/** + * ClickBench: a Benchmark For Analytical Databases + * + * See the project: https://github.com/ClickHouse/ClickBench + * Site: https://benchmark.clickhouse.com/ + */ +class ClickBenchSuite( + val masterUrl: String, + val actions: Array[Action], + val testConf: SparkConf, + val baselineConf: SparkConf, + val extraSparkConf: Map[String, String], + val logLevel: Level, + val errorOnMemLeak: Boolean, + val enableUi: Boolean, + val enableHsUi: Boolean, + val hsUiPort: Int, + val offHeapSize: String, + val disableAqe: Boolean, + val disableBhj: Boolean, + val disableWscg: Boolean, + val shufflePartitions: Int, + val minimumScanPartitions: Boolean) + extends Suite( + masterUrl, + actions, + testConf, + baselineConf, + extraSparkConf, + logLevel, + errorOnMemLeak, + enableUi, + enableHsUi, + hsUiPort, + offHeapSize, + disableAqe, + disableBhj, + disableWscg, + shufflePartitions, + minimumScanPartitions) { + import ClickBenchSuite._ + + override protected def historyWritePath(): String = HISTORY_WRITE_PATH + + override private[integration] def dataWritePath( + scale: Double, + genPartitionedData: Boolean): String = { + checkDataGenArgs(scale, genPartitionedData) + DATA_WRITE_PATH + } + + override private[integration] def createDataGen( + scale: Double, + genPartitionedData: Boolean): DataGen = { + new ClickBenchDataGen(sessionSwitcher.spark(), dataWritePath(scale, genPartitionedData)) + } + + override private[integration] def queryResource(): String = "/clickbench-queries" + + override private[integration] def allQueryIds(): Array[String] = ALL_QUERY_IDS + + override private[integration] def desc(): String = "ClickBench" + + override def tableCreator(): TableCreator = ClickBenchTableCreator +} + +private object ClickBenchSuite { + private val DATA_WRITE_PATH = "/tmp/clickbench-generated" + private val HISTORY_WRITE_PATH = "/tmp/clickbench-history" + private val ALL_QUERY_IDS = (1 to 43).map(i => s"q$i").toArray + + private def checkDataGenArgs(scale: Double, genPartitionedData: Boolean): Unit = { + assert(scale == 1.0D, "ClickBench suite doesn't support scale factor other than 1") + assert(!genPartitionedData, "ClickBench suite doesn't support generating partitioned data") + } +} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchTableCreator.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchTableCreator.scala new file mode 100644 index 000000000000..33eac38629ef --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchTableCreator.scala @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gluten.integration.clickbench + +import org.apache.gluten.integration.TableCreator +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.{AnalysisException, SparkSession} + +import java.io.File + +object ClickBenchTableCreator extends TableCreator { + private val TABLE_NAME = "hits" + private val SCHEMA: StructType = StructType.fromDDL(""" + |watchid bigint, + |javaenable smallint, + |title varchar(65535), + |goodevent smallint, + |eventtime timestamp, + |eventdate date, + |counterid int, + |clientip int, + |regionid int, + |userid bigint, + |counterclass smallint, + |os smallint, + |useragent smallint, + |url varchar(65535), + |referer varchar(65535), + |isrefresh smallint, + |referercategoryid smallint, + |refererregionid int, + |urlcategoryid smallint, + |urlregionid int, + |resolutionwidth smallint, + |resolutionheight smallint, + |resolutiondepth smallint, + |flashmajor smallint, + |flashminor smallint, + |flashminor2 varchar(65535), + |netmajor smallint, + |netminor smallint, + |useragentmajor smallint, + |useragentminor varchar(65535), + |cookieenable smallint, + |javascriptenable smallint, + |ismobile smallint, + |mobilephone smallint, + |mobilephonemodel varchar(65535), + |params varchar(65535), + |ipnetworkid int, + |traficsourceid smallint, + |searchengineid smallint, + |searchphrase varchar(65535), + |advengineid smallint, + |isartifical smallint, + |windowclientwidth smallint, + |windowclientheight smallint, + |clienttimezone smallint, + |clienteventtime timestamp, + |silverlightversion1 smallint, + |silverlightversion2 smallint, + |silverlightversion3 int, + |silverlightversion4 smallint, + |pagecharset varchar(65535), + |codeversion int, + |islink smallint, + |isdownload smallint, + |isnotbounce smallint, + |funiqid bigint, + |originalurl varchar(65535), + |hid int, + |isoldcounter smallint, + |isevent smallint, + |isparameter smallint, + |dontcounthits smallint, + |withhash smallint, + |hitcolor varchar(65535), + |localeventtime timestamp, + |age smallint, + |sex smallint, + |income smallint, + |interests smallint, + |robotness smallint, + |remoteip int, + |windowname int, + |openername int, + |historylength smallint, + |browserlanguage varchar(65535), + |browsercountry varchar(65535), + |socialnetwork varchar(65535), + |socialaction varchar(65535), + |httperror smallint, + |sendtiming int, + |dnstiming int, + |connecttiming int, + |responsestarttiming int, + |responseendtiming int, + |fetchtiming int, + |socialsourcenetworkid smallint, + |socialsourcepage varchar(65535), + |paramprice bigint, + |paramorderid varchar(65535), + |paramcurrency varchar(65535), + |paramcurrencyid smallint, + |openstatservicename varchar(65535), + |openstatcampaignid varchar(65535), + |openstatadid varchar(65535), + |openstatsourceid varchar(65535), + |utmsource varchar(65535), + |utmmedium varchar(65535), + |utmcampaign varchar(65535), + |utmcontent varchar(65535), + |utmterm varchar(65535), + |fromtag varchar(65535), + |hasgclid smallint, + |refererhash bigint, + |urlhash bigint, + |clid int + |""".stripMargin) + + override def create(spark: SparkSession, dataPath: String): Unit = { + val file = new File(dataPath + File.separator + ClickBenchDataGen.FILE_NAME) + if (spark.catalog.tableExists(TABLE_NAME)) { + println("Table exists: " + TABLE_NAME) + return + } + println("Creating catalog table: " + TABLE_NAME) + spark.catalog.createTable(TABLE_NAME, "parquet", SCHEMA, Map("path" -> file.getAbsolutePath)) + try { + spark.catalog.recoverPartitions(file.getName) + } catch { + case _: AnalysisException => + } + } +} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsDataGen.scala similarity index 99% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsDataGen.scala index 82d16dd90f1a..7d63fc67a999 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsDataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsDataGen.scala @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.ds +package org.apache.gluten.integration.ds -import org.apache.gluten.integration.tpc.{DataGen, ShimUtils, TypeModifier} +import org.apache.gluten.integration.{DataGen, ShimUtils, TypeModifier} import org.apache.spark.sql.{Column, Row, SaveMode, SparkSession} import org.apache.spark.sql.types._ diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala similarity index 78% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala index c703821c1b0f..339e89d5be71 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/ds/TpcdsSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala @@ -14,19 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.ds +package org.apache.gluten.integration.ds -import org.apache.gluten.integration.tpc.{Constants, DataGen, TpcSuite, TypeModifier} -import org.apache.gluten.integration.tpc.action.Action -import org.apache.gluten.integration.tpc.ds.TpcdsSuite.{ +import org.apache.gluten.integration.action.Action +import org.apache.gluten.integration.ds.TpcdsSuite.{ ALL_QUERY_IDS, HISTORY_WRITE_PATH, TPCDS_WRITE_PATH } - -import org.apache.spark.SparkConf - +import org.apache.gluten.integration.{DataGen, Suite, TableCreator, TypeModifier} import org.apache.log4j.Level +import org.apache.spark.SparkConf class TpcdsSuite( val masterUrl: String, @@ -45,7 +43,7 @@ class TpcdsSuite( val disableWscg: Boolean, val shufflePartitions: Int, val minimumScanPartitions: Boolean) - extends TpcSuite( + extends Suite( masterUrl, actions, testConf, @@ -65,28 +63,35 @@ class TpcdsSuite( override protected def historyWritePath(): String = HISTORY_WRITE_PATH - override private[tpc] def dataWritePath(scale: Double): String = TPCDS_WRITE_PATH + s"-$scale" + override private[integration] def dataWritePath( + scale: Double, + genPartitionedData: Boolean): String = + TPCDS_WRITE_PATH + s"-$scale" - override private[tpc] def createDataGen(scale: Double, genPartitionedData: Boolean): DataGen = + override private[integration] def createDataGen( + scale: Double, + genPartitionedData: Boolean): DataGen = new TpcdsDataGen( sessionSwitcher.spark(), scale, shufflePartitions, - dataWritePath(scale), + dataWritePath(scale, genPartitionedData), typeModifiers(), genPartitionedData) - override private[tpc] def queryResource(): String = { + override private[integration] def queryResource(): String = { "/tpcds-queries" } - override protected def typeModifiers(): List[TypeModifier] = { + private def typeModifiers(): List[TypeModifier] = { List() } - override private[tpc] def allQueryIds(): Array[String] = ALL_QUERY_IDS + override private[integration] def allQueryIds(): Array[String] = ALL_QUERY_IDS + + override private[integration] def desc(): String = "TPC-DS" - override private[tpc] def desc(): String = "TPC-DS" + override def tableCreator(): TableCreator = TableCreator.discoverSchema() } object TpcdsSuite { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchDataGen.scala similarity index 98% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchDataGen.scala index fa574f59c5d2..5223c61c99e0 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchDataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchDataGen.scala @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.h +package org.apache.gluten.integration.h -import org.apache.gluten.integration.tpc.{DataGen, ShimUtils, TypeModifier} +import org.apache.gluten.integration.{DataGen, ShimUtils, TypeModifier} import org.apache.spark.sql.{Row, SaveMode, SparkSession} import org.apache.spark.sql.types._ diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala similarity index 71% rename from tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala rename to tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala index 9fbd83dc2f66..29c299beebf1 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/tpc/h/TpchSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala @@ -14,15 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc.h - -import org.apache.gluten.integration.tpc.{DataGen, TpcSuite, TypeModifier} -import org.apache.gluten.integration.tpc.action.Action -import org.apache.gluten.integration.tpc.h.TpchSuite.{HISTORY_WRITE_PATH, TPCH_WRITE_PATH} - -import org.apache.spark.SparkConf +package org.apache.gluten.integration.h +import org.apache.gluten.integration.action.Action +import org.apache.gluten.integration.h.TpchSuite.{HISTORY_WRITE_PATH, TPCH_WRITE_PATH} +import org.apache.gluten.integration.{DataGen, Suite, TableCreator, TypeModifier} import org.apache.log4j.Level +import org.apache.spark.SparkConf class TpchSuite( val masterUrl: String, @@ -41,7 +39,7 @@ class TpchSuite( val disableWscg: Boolean, val shufflePartitions: Int, val minimumScanPartitions: Boolean) - extends TpcSuite( + extends Suite( masterUrl, actions, testConf, @@ -61,27 +59,34 @@ class TpchSuite( override protected def historyWritePath(): String = HISTORY_WRITE_PATH - override private[tpc] def dataWritePath(scale: Double): String = TPCH_WRITE_PATH + s"-$scale" + override private[integration] def dataWritePath( + scale: Double, + genPartitionedData: Boolean): String = + TPCH_WRITE_PATH + s"-$scale" - override private[tpc] def createDataGen(scale: Double, genPartitionedData: Boolean): DataGen = + override private[integration] def createDataGen( + scale: Double, + genPartitionedData: Boolean): DataGen = new TpchDataGen( sessionSwitcher.spark(), scale, shufflePartitions, - dataWritePath(scale), + dataWritePath(scale, genPartitionedData), typeModifiers()) - override private[tpc] def queryResource(): String = { + override private[integration] def queryResource(): String = { "/tpch-queries" } - override protected def typeModifiers(): List[TypeModifier] = { + private def typeModifiers(): List[TypeModifier] = { List() } - override private[tpc] def allQueryIds(): Array[String] = TpchSuite.ALL_QUERY_IDS + override private[integration] def allQueryIds(): Array[String] = TpchSuite.ALL_QUERY_IDS + + override private[integration] def desc(): String = "TPC-H" - override private[tpc] def desc(): String = "TPC-H" + override def tableCreator(): TableCreator = TableCreator.discoverSchema() } object TpchSuite { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala similarity index 98% rename from tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala rename to tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala index a5b699a1ae48..bb11a679f9eb 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/QueryRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala @@ -33,7 +33,7 @@ import java.io.ByteArrayOutputStream import java.nio.charset.StandardCharsets import java.util.concurrent.atomic.AtomicInteger -object QueryRunner { +object SparkQueryRunner { private val availableExecutorMetrics: Set[String] = Set( "JVMHeapMemory", "JVMOffHeapMemory", @@ -52,7 +52,7 @@ object QueryRunner { "ProcessTreeOtherVMemory", "ProcessTreeOtherRSSMemory") - def runTpcQuery( + def runQuery( spark: SparkSession, desc: String, queryPath: String, @@ -106,7 +106,7 @@ object QueryRunner { } private def resourceToString(resource: String): String = { - val inStream = QueryRunner.getClass.getResourceAsStream(resource) + val inStream = SparkQueryRunner.getClass.getResourceAsStream(resource) Preconditions.checkNotNull(inStream) val outStream = new ByteArrayOutputStream try { diff --git a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/tpc/TpcTest.java b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/CliTest.java similarity index 90% rename from tools/gluten-it/common/src/test/java/org/apache/gluten/integration/tpc/TpcTest.java rename to tools/gluten-it/common/src/test/java/org/apache/gluten/integration/CliTest.java index 2463429afdfb..804cf3a0a5af 100644 --- a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/tpc/TpcTest.java +++ b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/CliTest.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.integration.tpc; +package org.apache.gluten.integration; -public class TpcTest { +public class CliTest { public static void main(String[] args) { - Tpc.main(args); + Cli.main(args); } } diff --git a/tools/gluten-it/sbin/gluten-it.sh b/tools/gluten-it/sbin/gluten-it.sh index 5262df0541e3..fda117417936 100755 --- a/tools/gluten-it/sbin/gluten-it.sh +++ b/tools/gluten-it/sbin/gluten-it.sh @@ -47,4 +47,4 @@ $JAVA_HOME/bin/java $GLUTEN_IT_JVM_ARGS \ -Djdk.reflect.useDirectMethodHandle=false \ -Dio.netty.tryReflectionSetAccessible=true \ -cp $JAR_PATH \ - org.apache.gluten.integration.tpc.Tpc $@ + org.apache.gluten.integration.Cli $@ From 76168034983bb4bba055e7e5940d3558880ea3a8 Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Tue, 28 May 2024 15:46:35 +0800 Subject: [PATCH 161/402] [CORE] Only materialize subquery before doing transform (#5862) We transform subquery(e.g., dpp) during columanr rules which is not actually been executed, so we should not materialize subquery when replacing expression as it is not in concurrent. This pr wraps doTransform with transform to always do materialize subquery before doTransform, so that the subquries can be submitted in concurrent. --- .../CHHashAggregateExecTransformer.scala | 4 +- .../GlutenCustomAggExpressionSuite.scala | 2 +- .../benchmarks/CHParquetReadBenchmark.scala | 2 +- .../HashAggregateExecTransformer.scala | 4 +- .../gluten/execution/TopNTransformer.scala | 4 +- .../BasicPhysicalOperatorTransformer.scala | 6 +- .../execution/BasicScanExecTransformer.scala | 6 +- ...oadcastNestedLoopJoinExecTransformer.scala | 6 +- .../CartesianProductExecTransformer.scala | 6 +- .../execution/ExpandExecTransformer.scala | 4 +- .../GenerateExecTransformerBase.scala | 4 +- .../execution/JoinExecTransformer.scala | 6 +- .../gluten/execution/LimitTransformer.scala | 4 +- .../execution/SortExecTransformer.scala | 4 +- .../SortMergeJoinExecTransformer.scala | 6 +- .../execution/WholeStageTransformer.scala | 60 +- .../execution/WindowExecTransformer.scala | 4 +- .../WindowGroupLimitExecTransformer.scala | 4 +- .../execution/WriteFilesExecTransformer.scala | 4 +- .../expression/ExpressionConverter.scala | 21 +- .../ScalarSubqueryTransformer.scala | 21 +- .../columnar/enumerated/RemoveFilter.scala | 4 +- .../ColumnarCollapseTransformStages.scala | 2 +- .../python/EvalPythonExecTransformer.scala | 4 +- .../scalar-subquery-select.sql | 363 -------- .../scalar-subquery-select.sql.out | 791 ------------------ .../velox/VeloxSQLQueryTestSettings.scala | 6 +- .../utils/velox/VeloxTestSettings.scala | 2 - .../GlutenQueryExecutionErrorsSuite.scala | 11 - .../scalar-subquery-select.sql | 257 ------ .../scalar-subquery-select.sql.out | 614 -------------- .../velox/VeloxSQLQueryTestSettings.scala | 6 +- .../utils/velox/VeloxTestSettings.scala | 2 - .../GlutenQueryExecutionErrorsSuite.scala | 11 - 34 files changed, 88 insertions(+), 2167 deletions(-) delete mode 100644 gluten-ut/spark34/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql delete mode 100644 gluten-ut/spark34/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out delete mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql delete mode 100644 gluten-ut/spark35/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala index 4a4d345db1dd..7e688814381b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashAggregateExecTransformer.scala @@ -81,8 +81,8 @@ case class CHHashAggregateExecTransformer( } } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) val aggParams = new AggregationParams diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala index 3a2808e70f22..ba7d2c8f1935 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala @@ -91,7 +91,7 @@ class GlutenCustomAggExpressionSuite extends GlutenClickHouseTPCHAbstractSuite { assert(planExecs(3).isInstanceOf[HashAggregateExec]) val substraitContext = new SubstraitContext - planExecs(2).asInstanceOf[CHHashAggregateExecTransformer].doTransform(substraitContext) + planExecs(2).asInstanceOf[CHHashAggregateExecTransformer].transform(substraitContext) // Check the functions assert(substraitContext.registeredFunction.containsKey("custom_sum_double:req_fp64")) diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala index 5e802eeed1e4..dc1431fa64fa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala @@ -104,7 +104,7 @@ object CHParquetReadBenchmark extends SqlBasedBenchmark with CHSqlBasedBenchmark val scanTime = chFileScan.longMetric("scanTime") // Generate Substrait plan val substraitContext = new SubstraitContext - val transformContext = chFileScan.doTransform(substraitContext) + val transformContext = chFileScan.transform(substraitContext) val outNames = new java.util.ArrayList[String]() for (attr <- outputAttrs) { outNames.add(ConverterUtils.genColumnNameWithExprId(attr)) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala index 2f447572406b..01ab56881936 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala @@ -65,8 +65,8 @@ abstract class HashAggregateExecTransformer( super.output } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val aggParams = new AggregationParams val operatorId = context.nextOperatorId(this.nodeName) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala index 9df821fe10c5..c2d12415c78b 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala @@ -67,8 +67,8 @@ case class TopNTransformer( doNativeValidation(context, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) val relNode = getRelNode( diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala index e703295d0851..962ad6aca9d3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala @@ -129,8 +129,8 @@ abstract class FilterExecTransformerBase(val cond: Expression, val input: SparkP doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val remainingCondition = getRemainingCondition val operatorId = context.nextOperatorId(this.nodeName) if (remainingCondition == null) { @@ -190,7 +190,7 @@ case class ProjectExecTransformer private (projectList: Seq[NamedExpression], ch BackendsApiManager.getMetricsApiInstance.genProjectTransformerMetricsUpdater(metrics) override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) if ((projectList == null || projectList.isEmpty) && childCtx != null) { // The computing for this project is not needed. diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index b0bc0ea7b27d..2dd5aff766a9 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -80,7 +80,7 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource val numOutputVectors = longMetric("outputVectors") val scanTime = longMetric("scanTime") val substraitContext = new SubstraitContext - val transformContext = doTransform(substraitContext) + val transformContext = transform(substraitContext) val outNames = filteRedundantField(outputAttributes()).map(ConverterUtils.genColumnNameWithExprId).asJava val planNode = @@ -117,7 +117,7 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource } val substraitContext = new SubstraitContext - val relNode = doTransform(substraitContext).root + val relNode = transform(substraitContext).root doNativeValidation(substraitContext, relNode) } @@ -133,7 +133,7 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource } } - override def doTransform(context: SubstraitContext): TransformContext = { + override protected def doTransform(context: SubstraitContext): TransformContext = { val output = filteRedundantField(outputAttributes()) val typeNodes = ConverterUtils.collectAttributeTypeNodes(output) val nameList = ConverterUtils.collectAttributeNamesWithoutExprId(output) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala index 19a2ec8541e2..2f666a811dab 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala @@ -119,12 +119,12 @@ abstract class BroadcastNestedLoopJoinExecTransformer( } } - override def doTransform(context: SubstraitContext): TransformContext = { - val streamedPlanContext = streamedPlan.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val streamedPlanContext = streamedPlan.asInstanceOf[TransformSupport].transform(context) val (inputStreamedRelNode, inputStreamedOutput) = (streamedPlanContext.root, streamedPlanContext.outputAttributes) - val buildPlanContext = buildPlan.asInstanceOf[TransformSupport].doTransform(context) + val buildPlanContext = buildPlan.asInstanceOf[TransformSupport].transform(context) val (inputBuildRelNode, inputBuildOutput) = (buildPlanContext.root, buildPlanContext.outputAttributes) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala index c5a4a0eb81e8..91831f18493a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/CartesianProductExecTransformer.scala @@ -77,12 +77,12 @@ case class CartesianProductExecTransformer( BackendsApiManager.getMetricsApiInstance.genNestedLoopJoinTransformerMetricsUpdater(metrics) } - override def doTransform(context: SubstraitContext): TransformContext = { - val leftPlanContext = left.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val leftPlanContext = left.asInstanceOf[TransformSupport].transform(context) val (inputLeftRelNode, inputLeftOutput) = (leftPlanContext.root, leftPlanContext.outputAttributes) - val rightPlanContext = right.asInstanceOf[TransformSupport].doTransform(context) + val rightPlanContext = right.asInstanceOf[TransformSupport].transform(context) val (inputRightRelNode, inputRightOutput) = (rightPlanContext.root, rightPlanContext.outputAttributes) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala index aa98d88b29a8..362debb531ee 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala @@ -110,8 +110,8 @@ case class ExpandExecTransformer( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) if (projections == null || projections.isEmpty) { // The computing for this Expand is not needed. diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/GenerateExecTransformerBase.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/GenerateExecTransformerBase.scala index 5811f7b47fe4..b5c9b85aeb0d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/GenerateExecTransformerBase.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/GenerateExecTransformerBase.scala @@ -76,8 +76,8 @@ abstract class GenerateExecTransformerBase( doNativeValidation(context, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val relNode = getRelNode(context, childCtx.root, getGeneratorNode(context), validation = false) TransformContext(child.output, output, relNode) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala index e47ad8c7b1bd..0414c95aa918 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala @@ -227,12 +227,12 @@ trait HashJoinLikeExecTransformer extends BaseJoinExec with TransformSupport { doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val streamedPlanContext = streamedPlan.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val streamedPlanContext = streamedPlan.asInstanceOf[TransformSupport].transform(context) val (inputStreamedRelNode, inputStreamedOutput) = (streamedPlanContext.root, streamedPlanContext.outputAttributes) - val buildPlanContext = buildPlan.asInstanceOf[TransformSupport].doTransform(context) + val buildPlanContext = buildPlan.asInstanceOf[TransformSupport].transform(context) val (inputBuildRelNode, inputBuildOutput) = (buildPlanContext.root, buildPlanContext.outputAttributes) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/LimitTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/LimitTransformer.scala index 3379542ad1d6..8859844be48e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/LimitTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/LimitTransformer.scala @@ -53,8 +53,8 @@ case class LimitTransformer(child: SparkPlan, offset: Long, count: Long) doNativeValidation(context, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) val relNode = getRelNode(context, operatorId, offset, count, child.output, childCtx.root, false) TransformContext(child.output, child.output, relNode) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/SortExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SortExecTransformer.scala index 9e4a20d40ad4..f79dc69e680b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/SortExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SortExecTransformer.scala @@ -101,8 +101,8 @@ case class SortExecTransformer( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) if (sortOrder == null || sortOrder.isEmpty) { // The computing for this project is not needed. diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala index 5ca11a53c88a..98b3666f84ba 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala @@ -191,12 +191,12 @@ abstract class SortMergeJoinExecTransformerBase( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val streamedPlanContext = streamedPlan.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val streamedPlanContext = streamedPlan.asInstanceOf[TransformSupport].transform(context) val (inputStreamedRelNode, inputStreamedOutput) = (streamedPlanContext.root, streamedPlanContext.outputAttributes) - val bufferedPlanContext = bufferedPlan.asInstanceOf[TransformSupport].doTransform(context) + val bufferedPlanContext = bufferedPlan.asInstanceOf[TransformSupport].transform(context) val (inputBuildRelNode, inputBuildOutput) = (bufferedPlanContext.root, bufferedPlanContext.outputAttributes) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala index b809ac4bf1a4..ed691fc09613 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala @@ -68,7 +68,22 @@ trait TransformSupport extends GlutenPlan { */ def columnarInputRDDs: Seq[RDD[ColumnarBatch]] - def doTransform(context: SubstraitContext): TransformContext = { + final def transform(context: SubstraitContext): TransformContext = { + if (isCanonicalizedPlan) { + throw new IllegalStateException( + "A canonicalized plan is not supposed to be executed transform.") + } + if (TransformerState.underValidationState) { + doTransform(context) + } else { + // Materialize subquery first before going to do transform. + executeQuery { + doTransform(context) + } + } + } + + protected def doTransform(context: SubstraitContext): TransformContext = { throw new UnsupportedOperationException( s"This operator doesn't support doTransform with SubstraitContext.") } @@ -182,7 +197,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f val substraitContext = new SubstraitContext val childCtx = child .asInstanceOf[TransformSupport] - .doTransform(substraitContext) + .transform(substraitContext) if (childCtx == null) { throw new NullPointerException(s"WholeStageTransformer can't do Transform on $child") } @@ -216,8 +231,6 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f } def doWholeStageTransform(): WholeStageTransformContext = { - // invoke SparkPlan.prepare to do subquery preparation etc. - super.prepare() val context = generateWholeStageTransformContext() if (conf.getConf(GlutenConfig.CACHE_WHOLE_STAGE_TRANSFORMER_CONTEXT)) { wholeStageTransformerContext = Some(context) @@ -257,6 +270,12 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f override def doExecuteColumnar(): RDD[ColumnarBatch] = { val pipelineTime: SQLMetric = longMetric("pipelineTime") + // We should do transform first to make sure all subqueries are materialized + val wsCtx = GlutenTimeMetric.withMillisTime { + doWholeStageTransform() + }( + t => + logOnLevel(substraitPlanLogLevel, s"$nodeName generating the substrait plan took: $t ms.")) val inputRDDs = new ColumnarInputRDDsWrapper(columnarInputRDDs) // Check if BatchScan exists. val basicScanExecTransformers = findAllScanTransformers() @@ -271,22 +290,11 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f val allScanPartitions = basicScanExecTransformers.map(_.getPartitions) val allScanSplitInfos = getSplitInfosFromPartitions(basicScanExecTransformers, allScanPartitions) - - val (wsCtx, inputPartitions) = GlutenTimeMetric.withMillisTime { - val wsCtx = doWholeStageTransform() - val partitions = - BackendsApiManager.getIteratorApiInstance.genPartitions( - wsCtx, - allScanSplitInfos, - basicScanExecTransformers) - - (wsCtx, partitions) - }( - t => - logOnLevel( - substraitPlanLogLevel, - s"$nodeName generating the substrait plan took: $t ms.")) - + val inputPartitions = + BackendsApiManager.getIteratorApiInstance.genPartitions( + wsCtx, + allScanSplitInfos, + basicScanExecTransformers) val rdd = new GlutenWholeStageColumnarRDD( sparkContext, inputPartitions, @@ -321,22 +329,18 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f * GlutenDataFrameAggregateSuite) in these cases, separate RDDs takes care of SCAN as a * result, genFinalStageIterator rather than genFirstStageIterator will be invoked */ - val resCtx = GlutenTimeMetric.withMillisTime(doWholeStageTransform()) { - t => - logOnLevel(substraitPlanLogLevel, s"$nodeName generating the substrait plan took: $t ms.") - } new WholeStageZippedPartitionsRDD( sparkContext, inputRDDs, numaBindingInfo, sparkConf, - resCtx, + wsCtx, pipelineTime, BackendsApiManager.getMetricsApiInstance.metricsUpdatingFunction( child, - resCtx.substraitContext.registeredRelMap, - resCtx.substraitContext.registeredJoinParams, - resCtx.substraitContext.registeredAggregationParams + wsCtx.substraitContext.registeredRelMap, + wsCtx.substraitContext.registeredJoinParams, + wsCtx.substraitContext.registeredAggregationParams ), materializeInput ) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala index d7c3d3dd56a4..ef6a767b5604 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala @@ -179,8 +179,8 @@ case class WindowExecTransformer( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) if (windowExpression == null || windowExpression.isEmpty) { // The computing for this operator is not needed. diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WindowGroupLimitExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WindowGroupLimitExecTransformer.scala index bba79fa761e8..46a4e1aa4eee 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WindowGroupLimitExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WindowGroupLimitExecTransformer.scala @@ -146,8 +146,8 @@ case class WindowGroupLimitExecTransformer( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) val currRel = diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala index 7df4afa8a6c1..14d58bfa8377 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala @@ -161,8 +161,8 @@ case class WriteFilesExecTransformer( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) val currRel = getRelNode(context, getFinalChildOutput(), operatorId, childCtx.root, validation = false) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 2d514118ac30..b66ec89eaf2b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -654,7 +654,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { // or ColumnarBroadcastExchange was disabled. partitionFilters } else { - val newPartitionFilters = partitionFilters.map { + partitionFilters.map { case dynamicPruning: DynamicPruningExpression => dynamicPruning.transform { // Lookup inside subqueries for duplicate exchanges. @@ -723,25 +723,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { } case e: Expression => e } - updateSubqueryResult(newPartitionFilters) - newPartitionFilters - } - } - - private def updateSubqueryResult(partitionFilters: Seq[Expression]): Unit = { - // When it includes some DynamicPruningExpression, - // it needs to execute InSubqueryExec first, - // because doTransform path can't execute 'doExecuteColumnar' which will - // execute prepare subquery first. - partitionFilters.foreach { - case DynamicPruningExpression(inSubquery: InSubqueryExec) => - if (inSubquery.values().isEmpty) inSubquery.updateResult() - case e: Expression => - e.foreach { - case s: ScalarSubquery => s.updateResult() - case _ => - } - case _ => } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala index 0accf9ffd0f9..9508d27df73b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ScalarSubqueryTransformer.scala @@ -18,6 +18,7 @@ package org.apache.gluten.expression import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.ScalarSubquery @@ -30,21 +31,11 @@ case class ScalarSubqueryTransformer(substraitExprName: String, query: ScalarSub if (TransformerState.underValidationState) { return ExpressionBuilder.makeLiteral(null, query.dataType, true) } - // the first column in first row from `query`. - val rows = query.plan.executeCollect() - if (rows.length > 1) { - throw new IllegalStateException( - s"more than one row returned by a subquery used as an expression:\n${query.plan}") - } - val result: AnyRef = if (rows.length == 1) { - assert( - rows(0).numFields == 1, - s"Expects 1 field, but got ${rows(0).numFields}; something went wrong in analysis") - rows(0).get(0, query.dataType) - } else { - // If there is no rows returned, the result should be null. - null - } + // After https://github.com/apache/incubator-gluten/pull/5862, we do not need to execute + // subquery manually so the exception behavior is same with vanilla Spark. + // Note that, this code change is just for simplify. The subquery has already been materialized + // before doing transform. + val result = query.eval(InternalRow.empty) ExpressionBuilder.makeLiteral(result, query.dataType, result == null) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index 55b29cd56ff1..b980c24227d5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -75,8 +75,8 @@ object RemoveFilter extends RasRule[SparkPlan] { override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(newChild) override def outputPartitioning: Partitioning = child.outputPartitioning override def outputOrdering: Seq[SortOrder] = child.outputOrdering - override def doTransform(context: SubstraitContext): TransformContext = - child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = + child.asInstanceOf[TransformSupport].transform(context) override protected def doExecuteColumnar(): RDD[ColumnarBatch] = child.executeColumnar() } } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala index 23746846e9cf..c9bbf4e1c0eb 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala @@ -65,7 +65,7 @@ case class InputIteratorTransformer(child: SparkPlan) extends UnaryTransformSupp child.doExecuteBroadcast() } - override def doTransform(context: SubstraitContext): TransformContext = { + override protected def doTransform(context: SubstraitContext): TransformContext = { val operatorId = context.nextOperatorId(nodeName) val readRel = RelBuilder.makeReadRelForInputIterator(child.output.asJava, context, operatorId) TransformContext(output, output, readRel) diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecTransformer.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecTransformer.scala index 43dd2f453803..ecedc1bae01c 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExecTransformer.scala @@ -84,8 +84,8 @@ case class EvalPythonExecTransformer( doNativeValidation(context, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val args = context.registeredFunction val operatorId = context.nextOperatorId(this.nodeName) val expressionNodes = new JArrayList[ExpressionNode] diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql b/gluten-ut/spark34/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql deleted file mode 100644 index 48d1594fa51a..000000000000 --- a/gluten-ut/spark34/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql +++ /dev/null @@ -1,363 +0,0 @@ --- A test suite for scalar subquery in SELECT clause - -create temporary view t1 as select * from values - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 00:00:00.000', date '2014-04-04'), - ('val1b', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1a', 16S, 12, 21L, float(15.0), 20D, 20E2, timestamp '2014-06-04 01:02:00.001', date '2014-06-04'), - ('val1a', 16S, 12, 10L, float(15.0), 20D, 20E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:02:00.001', date '2014-05-05'), - ('val1d', null, 16, 22L, float(17.0), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', null), - ('val1d', null, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-07-04 01:02:00.001', null), - ('val1e', 10S, null, 25L, float(17.0), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-09-04 01:02:00.001', date '2014-09-04'), - ('val1d', 10S, null, 12L, float(17.0), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:02:00.001', date '2014-04-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04') - as t1(t1a, t1b, t1c, t1d, t1e, t1f, t1g, t1h, t1i); - -create temporary view t2 as select * from values - ('val2a', 6S, 12, 14L, float(15), 20D, 20E2, timestamp '2014-04-04 01:01:00.000', date '2014-04-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 119L, float(17), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1c', 12S, 16, 219L, float(17), 25D, 26E2, timestamp '2016-05-04 01:01:00.000', date '2016-05-04'), - ('val1b', null, 16, 319L, float(17), 25D, 26E2, timestamp '2017-05-04 01:01:00.000', null), - ('val2e', 8S, null, 419L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1f', 19S, null, 519L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 12S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-05'), - ('val1e', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:01:00.000', date '2014-09-04'), - ('val1f', 19S, null, 19L, float(17), 25D, 26E2, timestamp '2014-10-04 01:01:00.000', date '2014-10-04'), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', null) - as t2(t2a, t2b, t2c, t2d, t2e, t2f, t2g, t2h, t2i); - -create temporary view t3 as select * from values - ('val3a', 6S, 12, 110L, float(15), 20D, 20E2, timestamp '2014-04-04 01:02:00.000', date '2014-04-04'), - ('val3a', 6S, 12, 10L, float(15), 20D, 20E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 219L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 319L, float(17), 25D, 26E2, timestamp '2014-06-04 01:02:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:02:00.000', date '2014-07-04'), - ('val3c', 17S, 16, 519L, float(17), 25D, 26E2, timestamp '2014-08-04 01:02:00.000', date '2014-08-04'), - ('val3c', 17S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:02:00.000', date '2014-09-05'), - ('val1b', null, 16, 419L, float(17), 25D, 26E2, timestamp '2014-10-04 01:02:00.000', null), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-11-04 01:02:00.000', null), - ('val3b', 8S, null, 719L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val3b', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2015-05-04 01:02:00.000', date '2015-05-04') - as t3(t3a, t3b, t3c, t3d, t3e, t3f, t3g, t3h, t3i); - --- Group 1: scalar subquery in SELECT clause --- no correlation --- TC 01.01 --- more than one scalar subquery -SELECT (SELECT min(t3d) FROM t3) min_t3d, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c'; - --- TC 01.02 --- scalar subquery in an IN subquery -SELECT t1a, count(*) -FROM t1 -WHERE t1c IN (SELECT (SELECT min(t3c) FROM t3) - FROM t2 - GROUP BY t2g - HAVING count(*) > 1) -GROUP BY t1a; - --- TC 01.03 --- under a set op -SELECT (SELECT min(t3d) FROM t3) min_t3d, - null -FROM t1 -WHERE t1a = 'val1c' -UNION -SELECT null, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c'; - --- TC 01.04 -SELECT (SELECT min(t3c) FROM t3) min_t3d -FROM t1 -WHERE t1a = 'val1a' -INTERSECT -SELECT (SELECT min(t2c) FROM t2) min_t2d -FROM t1 -WHERE t1a = 'val1d'; - --- TC 01.05 -SELECT q1.t1a, q2.t2a, q1.min_t3d, q2.avg_t3d -FROM (SELECT t1a, (SELECT min(t3d) FROM t3) min_t3d - FROM t1 - WHERE t1a IN ('val1e', 'val1c')) q1 - FULL OUTER JOIN - (SELECT t2a, (SELECT avg(t3d) FROM t3) avg_t3d - FROM t2 - WHERE t2a IN ('val1c', 'val2a')) q2 -ON q1.t1a = q2.t2a -AND q1.min_t3d < q2.avg_t3d; - --- Group 2: scalar subquery in SELECT clause --- with correlation --- TC 02.01 -SELECT (SELECT min(t3d) FROM t3 WHERE t3.t3a = t1.t1a) min_t3d, - (SELECT max(t2h) FROM t2 WHERE t2.t2a = t1.t1a) max_t2h -FROM t1 -WHERE t1a = 'val1b'; - --- TC 02.02 -SELECT (SELECT min(t3d) FROM t3 WHERE t3a = t1a) min_t3d -FROM t1 -WHERE t1a = 'val1b' -MINUS -SELECT (SELECT min(t3d) FROM t3) abs_min_t3d -FROM t1 -WHERE t1a = 'val1b'; - --- TC 02.03 -SELECT t1a, t1b -FROM t1 -WHERE NOT EXISTS (SELECT (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) dummy - FROM t3 - WHERE t3b < (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) - AND t3a = t1a); - --- SPARK-34876: Non-nullable aggregates should not return NULL in a correlated subquery -SELECT t1a, - (SELECT count(t2d) FROM t2 WHERE t2a = t1a) count_t2, - (SELECT count_if(t2d > 0) FROM t2 WHERE t2a = t1a) count_if_t2, - (SELECT approx_count_distinct(t2d) FROM t2 WHERE t2a = t1a) approx_count_distinct_t2, - (SELECT collect_list(t2d) FROM t2 WHERE t2a = t1a) collect_list_t2, - (SELECT sort_array(collect_set(t2d)) FROM t2 WHERE t2a = t1a) collect_set_t2, - (SELECT hex(count_min_sketch(t2d, 0.5d, 0.5d, 1)) FROM t2 WHERE t2a = t1a) collect_set_t2 -FROM t1; - --- SPARK-36028: Allow Project to host outer references in scalar subqueries -SELECT t1c, (SELECT t1c) FROM t1; -SELECT t1c, (SELECT t1c WHERE t1c = 8) FROM t1; -SELECT t1c, t1d, (SELECT c + d FROM (SELECT t1c AS c, t1d AS d)) FROM t1; -SELECT t1c, (SELECT SUM(c) FROM (SELECT t1c AS c)) FROM t1; -SELECT t1a, (SELECT SUM(t2b) FROM t2 JOIN (SELECT t1a AS a) ON t2a = a) FROM t1; - --- CTE in correlated scalar subqueries -CREATE OR REPLACE TEMPORARY VIEW t1 AS VALUES (0, 1), (1, 2) t1(c1, c2); -CREATE OR REPLACE TEMPORARY VIEW t2 AS VALUES (0, 2), (0, 3) t2(c1, c2); - --- Single row subquery -SELECT c1, (WITH t AS (SELECT 1 AS a) SELECT a + c1 FROM t) FROM t1; --- Correlation in CTE. -SELECT c1, (WITH t AS (SELECT * FROM t2 WHERE c1 = t1.c1) SELECT SUM(c2) FROM t) FROM t1; --- Multiple CTE definitions. -SELECT c1, ( - WITH t3 AS (SELECT c1 + 1 AS c1, c2 + 1 AS c2 FROM t2), - t4 AS (SELECT * FROM t3 WHERE t1.c1 = c1) - SELECT SUM(c2) FROM t4 -) FROM t1; --- Multiple CTE references. -SELECT c1, ( - WITH t AS (SELECT * FROM t2) - SELECT SUM(c2) FROM (SELECT c1, c2 FROM t UNION SELECT c2, c1 FROM t) r(c1, c2) - WHERE c1 = t1.c1 -) FROM t1; --- Reference CTE in both the main query and the subquery. -WITH v AS (SELECT * FROM t2) -SELECT * FROM t1 WHERE c1 > ( - WITH t AS (SELECT * FROM t2) - SELECT COUNT(*) FROM v WHERE c1 = t1.c1 AND c1 > (SELECT SUM(c2) FROM t WHERE c1 = v.c1) -); --- Single row subquery that references CTE in the main query. -WITH t AS (SELECT 1 AS a) -SELECT c1, (SELECT a FROM t WHERE a = c1) FROM t1; --- Multiple CTE references with non-deterministic CTEs. -WITH -v1 AS (SELECT c1, c2, rand(0) c3 FROM t1), -v2 AS (SELECT c1, c2, rand(0) c4 FROM v1 WHERE c3 IN (SELECT c3 FROM v1)) -SELECT c1, ( - WITH v3 AS (SELECT c1, c2, rand(0) c5 FROM t2) - SELECT COUNT(*) FROM ( - SELECT * FROM v2 WHERE c1 > 0 - UNION SELECT * FROM v2 WHERE c2 > 0 - UNION SELECT * FROM v3 WHERE c2 > 0 - ) WHERE c1 = v1.c1 -) FROM v1; - --- Multi-value subquery error -SELECT (SELECT a FROM (SELECT 1 AS a UNION ALL SELECT 2 AS a) t) AS b; - --- SPARK-36114: Support correlated non-equality predicates -CREATE OR REPLACE TEMP VIEW t1(c1, c2) AS (VALUES (0, 1), (1, 2)); -CREATE OR REPLACE TEMP VIEW t2(c1, c2) AS (VALUES (0, 2), (0, 3)); - --- Neumann example Q2 -CREATE OR REPLACE TEMP VIEW students(id, name, major, year) AS (VALUES - (0, 'A', 'CS', 2022), - (1, 'B', 'CS', 2022), - (2, 'C', 'Math', 2022)); -CREATE OR REPLACE TEMP VIEW exams(sid, course, curriculum, grade, date) AS (VALUES - (0, 'C1', 'CS', 4, 2020), - (0, 'C2', 'CS', 3, 2021), - (1, 'C1', 'CS', 2, 2020), - (1, 'C2', 'CS', 1, 2021)); - -SELECT students.name, exams.course -FROM students, exams -WHERE students.id = exams.sid - AND (students.major = 'CS' OR students.major = 'Games Eng') - AND exams.grade >= ( - SELECT avg(exams.grade) + 1 - FROM exams - WHERE students.id = exams.sid - OR (exams.curriculum = students.major AND students.year > exams.date)); - --- Correlated non-equality predicates -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 > t2.c1) FROM t1; -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 >= t2.c1 AND t1.c2 < t2.c2) FROM t1; - --- Correlated non-equality predicates with the COUNT bug. -SELECT (SELECT count(*) FROM t2 WHERE t1.c1 > t2.c1) FROM t1; - --- Correlated equality predicates that are not supported after SPARK-35080 -SELECT c, ( - SELECT count(*) - FROM (VALUES ('ab'), ('abc'), ('bc')) t2(c) - WHERE t1.c = substring(t2.c, 1, 1) -) FROM (VALUES ('a'), ('b')) t1(c); - -SELECT c, ( - SELECT count(*) - FROM (VALUES (0, 6), (1, 5), (2, 4), (3, 3)) t1(a, b) - WHERE a + b = c -) FROM (VALUES (6)) t2(c); - --- SPARK-43156: scalar subquery with Literal result like `COUNT(1) is null` -SELECT *, (SELECT count(1) is null FROM t2 WHERE t1.c1 = t2.c1) FROM t1; - -select (select f from (select false as f, max(c2) from t1 where t1.c1 = t1.c1)) from t2; - --- Set operations in correlation path - -CREATE OR REPLACE TEMP VIEW t0(t0a, t0b) AS VALUES (1, 1), (2, 0); -CREATE OR REPLACE TEMP VIEW t1(t1a, t1b, t1c) AS VALUES (1, 1, 3); -CREATE OR REPLACE TEMP VIEW t2(t2a, t2b, t2c) AS VALUES (1, 1, 5), (2, 2, 7); - -SELECT t0a, (SELECT sum(c) FROM - (SELECT t1c as c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t2c as c - FROM t2 - WHERE t2b = t0b) -) -FROM t0; - -SELECT t0a, (SELECT sum(c) FROM - (SELECT t1c as c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t2c as c - FROM t2 - WHERE t2a = t0a) -) -FROM t0; - -SELECT t0a, (SELECT sum(c) FROM - (SELECT t1c as c - FROM t1 - WHERE t1a > t0a - UNION ALL - SELECT t2c as c - FROM t2 - WHERE t2b <= t0b) -) -FROM t0; - -SELECT t0a, (SELECT sum(t1c) FROM - (SELECT t1c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0; - -SELECT t0a, (SELECT sum(t1c) FROM - (SELECT t1c - FROM t1 - WHERE t1a = t0a - UNION DISTINCT - SELECT t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0; - --- Tests for column aliasing -SELECT t0a, (SELECT sum(t1a + 3 * t1b + 5 * t1c) FROM - (SELECT t1c as t1a, t1a as t1b, t0a as t1c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t0a as t2b, t2c as t1a, t0b as t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0; - --- Test handling of COUNT bug -SELECT t0a, (SELECT count(t1c) FROM - (SELECT t1c - FROM t1 - WHERE t1a = t0a - UNION DISTINCT - SELECT t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0; - --- Correlated references in project -SELECT t0a, (SELECT sum(d) FROM - (SELECT t1a - t0a as d - FROM t1 - UNION ALL - SELECT t2a - t0a as d - FROM t2) -) -FROM t0; - --- Correlated references in aggregate - unsupported -SELECT t0a, (SELECT sum(d) FROM - (SELECT sum(t0a) as d - FROM t1 - UNION ALL - SELECT sum(t2a) + t0a as d - FROM t2) -) -FROM t0; - --- SPARK-43760: the result of the subquery can be NULL. -select * -from -( - select t1.id c1, ( - select sum(c) - from ( - select t2.id * t2.id c - from range (1, 2) t2 where t1.id = t2.id - group by t2.id - ) - ) c2 - from range (1, 3) t1 -) t -where t.c2 is not null; diff --git a/gluten-ut/spark34/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out b/gluten-ut/spark34/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out deleted file mode 100644 index 088359d39b86..000000000000 --- a/gluten-ut/spark34/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out +++ /dev/null @@ -1,791 +0,0 @@ --- Automatically generated by GlutenSQLQueryTestSuite --- Number of queries: 52 - - --- !query -create temporary view t1 as select * from values - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 00:00:00.000', date '2014-04-04'), - ('val1b', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1a', 16S, 12, 21L, float(15.0), 20D, 20E2, timestamp '2014-06-04 01:02:00.001', date '2014-06-04'), - ('val1a', 16S, 12, 10L, float(15.0), 20D, 20E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:02:00.001', date '2014-05-05'), - ('val1d', null, 16, 22L, float(17.0), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', null), - ('val1d', null, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-07-04 01:02:00.001', null), - ('val1e', 10S, null, 25L, float(17.0), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-09-04 01:02:00.001', date '2014-09-04'), - ('val1d', 10S, null, 12L, float(17.0), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:02:00.001', date '2014-04-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04') - as t1(t1a, t1b, t1c, t1d, t1e, t1f, t1g, t1h, t1i) --- !query schema -struct<> --- !query output - - - --- !query -create temporary view t2 as select * from values - ('val2a', 6S, 12, 14L, float(15), 20D, 20E2, timestamp '2014-04-04 01:01:00.000', date '2014-04-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 119L, float(17), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1c', 12S, 16, 219L, float(17), 25D, 26E2, timestamp '2016-05-04 01:01:00.000', date '2016-05-04'), - ('val1b', null, 16, 319L, float(17), 25D, 26E2, timestamp '2017-05-04 01:01:00.000', null), - ('val2e', 8S, null, 419L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1f', 19S, null, 519L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 12S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-05'), - ('val1e', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:01:00.000', date '2014-09-04'), - ('val1f', 19S, null, 19L, float(17), 25D, 26E2, timestamp '2014-10-04 01:01:00.000', date '2014-10-04'), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', null) - as t2(t2a, t2b, t2c, t2d, t2e, t2f, t2g, t2h, t2i) --- !query schema -struct<> --- !query output - - - --- !query -create temporary view t3 as select * from values - ('val3a', 6S, 12, 110L, float(15), 20D, 20E2, timestamp '2014-04-04 01:02:00.000', date '2014-04-04'), - ('val3a', 6S, 12, 10L, float(15), 20D, 20E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 219L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 319L, float(17), 25D, 26E2, timestamp '2014-06-04 01:02:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:02:00.000', date '2014-07-04'), - ('val3c', 17S, 16, 519L, float(17), 25D, 26E2, timestamp '2014-08-04 01:02:00.000', date '2014-08-04'), - ('val3c', 17S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:02:00.000', date '2014-09-05'), - ('val1b', null, 16, 419L, float(17), 25D, 26E2, timestamp '2014-10-04 01:02:00.000', null), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-11-04 01:02:00.000', null), - ('val3b', 8S, null, 719L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val3b', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2015-05-04 01:02:00.000', date '2015-05-04') - as t3(t3a, t3b, t3c, t3d, t3e, t3f, t3g, t3h, t3i) --- !query schema -struct<> --- !query output - - - --- !query -SELECT (SELECT min(t3d) FROM t3) min_t3d, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c' --- !query schema -struct --- !query output -10 2017-05-04 01:01:00 - - --- !query -SELECT t1a, count(*) -FROM t1 -WHERE t1c IN (SELECT (SELECT min(t3c) FROM t3) - FROM t2 - GROUP BY t2g - HAVING count(*) > 1) -GROUP BY t1a --- !query schema -struct --- !query output -val1a 2 - - --- !query -SELECT (SELECT min(t3d) FROM t3) min_t3d, - null -FROM t1 -WHERE t1a = 'val1c' -UNION -SELECT null, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c' --- !query schema -struct --- !query output -10 NULL -NULL 2017-05-04 01:01:00 - - --- !query -SELECT (SELECT min(t3c) FROM t3) min_t3d -FROM t1 -WHERE t1a = 'val1a' -INTERSECT -SELECT (SELECT min(t2c) FROM t2) min_t2d -FROM t1 -WHERE t1a = 'val1d' --- !query schema -struct --- !query output -12 - - --- !query -SELECT q1.t1a, q2.t2a, q1.min_t3d, q2.avg_t3d -FROM (SELECT t1a, (SELECT min(t3d) FROM t3) min_t3d - FROM t1 - WHERE t1a IN ('val1e', 'val1c')) q1 - FULL OUTER JOIN - (SELECT t2a, (SELECT avg(t3d) FROM t3) avg_t3d - FROM t2 - WHERE t2a IN ('val1c', 'val2a')) q2 -ON q1.t1a = q2.t2a -AND q1.min_t3d < q2.avg_t3d --- !query schema -struct --- !query output -NULL val2a NULL 200.83333333333334 -val1c val1c 10 200.83333333333334 -val1c val1c 10 200.83333333333334 -val1e NULL 10 NULL -val1e NULL 10 NULL -val1e NULL 10 NULL - - --- !query -SELECT (SELECT min(t3d) FROM t3 WHERE t3.t3a = t1.t1a) min_t3d, - (SELECT max(t2h) FROM t2 WHERE t2.t2a = t1.t1a) max_t2h -FROM t1 -WHERE t1a = 'val1b' --- !query schema -struct --- !query output -19 2017-05-04 01:01:00 - - --- !query -SELECT (SELECT min(t3d) FROM t3 WHERE t3a = t1a) min_t3d -FROM t1 -WHERE t1a = 'val1b' -MINUS -SELECT (SELECT min(t3d) FROM t3) abs_min_t3d -FROM t1 -WHERE t1a = 'val1b' --- !query schema -struct --- !query output -19 - - --- !query -SELECT t1a, t1b -FROM t1 -WHERE NOT EXISTS (SELECT (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) dummy - FROM t3 - WHERE t3b < (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) - AND t3a = t1a) --- !query schema -struct --- !query output -val1a 16 -val1a 16 -val1a 6 -val1a 6 -val1c 8 -val1d 10 -val1d NULL -val1d NULL -val1e 10 -val1e 10 -val1e 10 - - --- !query -SELECT t1a, - (SELECT count(t2d) FROM t2 WHERE t2a = t1a) count_t2, - (SELECT count_if(t2d > 0) FROM t2 WHERE t2a = t1a) count_if_t2, - (SELECT approx_count_distinct(t2d) FROM t2 WHERE t2a = t1a) approx_count_distinct_t2, - (SELECT collect_list(t2d) FROM t2 WHERE t2a = t1a) collect_list_t2, - (SELECT sort_array(collect_set(t2d)) FROM t2 WHERE t2a = t1a) collect_set_t2, - (SELECT hex(count_min_sketch(t2d, 0.5d, 0.5d, 1)) FROM t2 WHERE t2a = t1a) collect_set_t2 -FROM t1 --- !query schema -struct,collect_set_t2:array,collect_set_t2:string> --- !query output -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1b 6 6 3 [19,119,319,19,19,19] [19,119,319] 0000000100000000000000060000000100000004000000005D8D6AB90000000000000000000000000000000400000000000000010000000000000001 -val1c 2 2 2 [219,19] [19,219] 0000000100000000000000020000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000001 -val1d 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1d 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1d 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1e 1 1 1 [19] [19] 0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000 -val1e 1 1 1 [19] [19] 0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000 -val1e 1 1 1 [19] [19] 0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000 - - --- !query -SELECT t1c, (SELECT t1c) FROM t1 --- !query schema -struct --- !query output -12 12 -12 12 -16 16 -16 16 -16 16 -16 16 -8 8 -8 8 -NULL NULL -NULL NULL -NULL NULL -NULL NULL - - --- !query -SELECT t1c, (SELECT t1c WHERE t1c = 8) FROM t1 --- !query schema -struct --- !query output -12 NULL -12 NULL -16 NULL -16 NULL -16 NULL -16 NULL -8 8 -8 8 -NULL NULL -NULL NULL -NULL NULL -NULL NULL - - --- !query -SELECT t1c, t1d, (SELECT c + d FROM (SELECT t1c AS c, t1d AS d)) FROM t1 --- !query schema -struct --- !query output -12 10 22 -12 21 33 -16 19 35 -16 19 35 -16 19 35 -16 22 38 -8 10 18 -8 10 18 -NULL 12 NULL -NULL 19 NULL -NULL 19 NULL -NULL 25 NULL - - --- !query -SELECT t1c, (SELECT SUM(c) FROM (SELECT t1c AS c)) FROM t1 --- !query schema -struct --- !query output -12 12 -12 12 -16 16 -16 16 -16 16 -16 16 -8 8 -8 8 -NULL NULL -NULL NULL -NULL NULL -NULL NULL - - --- !query -SELECT t1a, (SELECT SUM(t2b) FROM t2 JOIN (SELECT t1a AS a) ON t2a = a) FROM t1 --- !query schema -struct --- !query output -val1a NULL -val1a NULL -val1a NULL -val1a NULL -val1b 36 -val1c 24 -val1d NULL -val1d NULL -val1d NULL -val1e 8 -val1e 8 -val1e 8 - - --- !query -CREATE OR REPLACE TEMPORARY VIEW t1 AS VALUES (0, 1), (1, 2) t1(c1, c2) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMPORARY VIEW t2 AS VALUES (0, 2), (0, 3) t2(c1, c2) --- !query schema -struct<> --- !query output - - - --- !query -SELECT c1, (WITH t AS (SELECT 1 AS a) SELECT a + c1 FROM t) FROM t1 --- !query schema -struct --- !query output -0 1 -1 2 - - --- !query -SELECT c1, (WITH t AS (SELECT * FROM t2 WHERE c1 = t1.c1) SELECT SUM(c2) FROM t) FROM t1 --- !query schema -struct --- !query output -0 5 -1 NULL - - --- !query -SELECT c1, ( - WITH t3 AS (SELECT c1 + 1 AS c1, c2 + 1 AS c2 FROM t2), - t4 AS (SELECT * FROM t3 WHERE t1.c1 = c1) - SELECT SUM(c2) FROM t4 -) FROM t1 --- !query schema -struct --- !query output -0 NULL -1 7 - - --- !query -SELECT c1, ( - WITH t AS (SELECT * FROM t2) - SELECT SUM(c2) FROM (SELECT c1, c2 FROM t UNION SELECT c2, c1 FROM t) r(c1, c2) - WHERE c1 = t1.c1 -) FROM t1 --- !query schema -struct --- !query output -0 5 -1 NULL - - --- !query -WITH v AS (SELECT * FROM t2) -SELECT * FROM t1 WHERE c1 > ( - WITH t AS (SELECT * FROM t2) - SELECT COUNT(*) FROM v WHERE c1 = t1.c1 AND c1 > (SELECT SUM(c2) FROM t WHERE c1 = v.c1) -) --- !query schema -struct --- !query output -1 2 - - --- !query -WITH t AS (SELECT 1 AS a) -SELECT c1, (SELECT a FROM t WHERE a = c1) FROM t1 --- !query schema -struct --- !query output -0 NULL -1 1 - - --- !query -WITH -v1 AS (SELECT c1, c2, rand(0) c3 FROM t1), -v2 AS (SELECT c1, c2, rand(0) c4 FROM v1 WHERE c3 IN (SELECT c3 FROM v1)) -SELECT c1, ( - WITH v3 AS (SELECT c1, c2, rand(0) c5 FROM t2) - SELECT COUNT(*) FROM ( - SELECT * FROM v2 WHERE c1 > 0 - UNION SELECT * FROM v2 WHERE c2 > 0 - UNION SELECT * FROM v3 WHERE c2 > 0 - ) WHERE c1 = v1.c1 -) FROM v1 --- !query schema -struct --- !query output -0 3 -1 1 - - --- !query -SELECT (SELECT a FROM (SELECT 1 AS a UNION ALL SELECT 2 AS a) t) AS b --- !query schema -struct<> --- !query output -java.lang.IllegalStateException -more than one row returned by a subquery used as an expression: -Subquery subquery#1, [id=#2] -+- AdaptiveSparkPlan isFinalPlan=true - +- == Final Plan == - VeloxColumnarToRowExec - +- ColumnarUnion - :- ProjectExecTransformer [1 AS a#3] - : +- InputIteratorTransformer[fake_column#4] - : +- InputAdapter - : +- RowToVeloxColumnar - : +- Scan OneRowRelation[fake_column#4] - +- ProjectExecTransformer [2 AS a#5] - +- InputIteratorTransformer[fake_column#6] - +- InputAdapter - +- RowToVeloxColumnar - +- Scan OneRowRelation[fake_column#6] - +- == Initial Plan == - Union - :- Project [1 AS a#3] - : +- Scan OneRowRelation[] - +- Project [2 AS a#5] - +- Scan OneRowRelation[] - - --- !query -CREATE OR REPLACE TEMP VIEW t1(c1, c2) AS (VALUES (0, 1), (1, 2)) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW t2(c1, c2) AS (VALUES (0, 2), (0, 3)) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW students(id, name, major, year) AS (VALUES - (0, 'A', 'CS', 2022), - (1, 'B', 'CS', 2022), - (2, 'C', 'Math', 2022)) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW exams(sid, course, curriculum, grade, date) AS (VALUES - (0, 'C1', 'CS', 4, 2020), - (0, 'C2', 'CS', 3, 2021), - (1, 'C1', 'CS', 2, 2020), - (1, 'C2', 'CS', 1, 2021)) --- !query schema -struct<> --- !query output - - - --- !query -SELECT students.name, exams.course -FROM students, exams -WHERE students.id = exams.sid - AND (students.major = 'CS' OR students.major = 'Games Eng') - AND exams.grade >= ( - SELECT avg(exams.grade) + 1 - FROM exams - WHERE students.id = exams.sid - OR (exams.curriculum = students.major AND students.year > exams.date)) --- !query schema -struct --- !query output -A C1 - - --- !query -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 > t2.c1) FROM t1 --- !query schema -struct --- !query output -2 -NULL - - --- !query -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 >= t2.c1 AND t1.c2 < t2.c2) FROM t1 --- !query schema -struct --- !query output -2 -3 - - --- !query -SELECT (SELECT count(*) FROM t2 WHERE t1.c1 > t2.c1) FROM t1 --- !query schema -struct --- !query output -0 -2 - - --- !query -SELECT c, ( - SELECT count(*) - FROM (VALUES ('ab'), ('abc'), ('bc')) t2(c) - WHERE t1.c = substring(t2.c, 1, 1) -) FROM (VALUES ('a'), ('b')) t1(c) --- !query schema -struct --- !query output -a 2 -b 1 - - --- !query -SELECT c, ( - SELECT count(*) - FROM (VALUES (0, 6), (1, 5), (2, 4), (3, 3)) t1(a, b) - WHERE a + b = c -) FROM (VALUES (6)) t2(c) --- !query schema -struct --- !query output -6 4 - - --- !query -SELECT *, (SELECT count(1) is null FROM t2 WHERE t1.c1 = t2.c1) FROM t1 --- !query schema -struct --- !query output -0 1 false -1 2 false - - --- !query -select (select f from (select false as f, max(c2) from t1 where t1.c1 = t1.c1)) from t2 --- !query schema -struct --- !query output -false -false - - --- !query -CREATE OR REPLACE TEMP VIEW t0(t0a, t0b) AS VALUES (1, 1), (2, 0) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW t1(t1a, t1b, t1c) AS VALUES (1, 1, 3) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW t2(t2a, t2b, t2c) AS VALUES (1, 1, 5), (2, 2, 7) --- !query schema -struct<> --- !query output - - - --- !query -SELECT t0a, (SELECT sum(c) FROM - (SELECT t1c as c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t2c as c - FROM t2 - WHERE t2b = t0b) -) -FROM t0 --- !query schema -struct --- !query output -1 8 -2 NULL - - --- !query -SELECT t0a, (SELECT sum(c) FROM - (SELECT t1c as c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t2c as c - FROM t2 - WHERE t2a = t0a) -) -FROM t0 --- !query schema -struct --- !query output -1 8 -2 7 - - --- !query -SELECT t0a, (SELECT sum(c) FROM - (SELECT t1c as c - FROM t1 - WHERE t1a > t0a - UNION ALL - SELECT t2c as c - FROM t2 - WHERE t2b <= t0b) -) -FROM t0 --- !query schema -struct --- !query output -1 5 -2 NULL - - --- !query -SELECT t0a, (SELECT sum(t1c) FROM - (SELECT t1c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0 --- !query schema -struct --- !query output -1 8 -2 NULL - - --- !query -SELECT t0a, (SELECT sum(t1c) FROM - (SELECT t1c - FROM t1 - WHERE t1a = t0a - UNION DISTINCT - SELECT t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0 --- !query schema -struct --- !query output -1 8 -2 NULL - - --- !query -SELECT t0a, (SELECT sum(t1a + 3 * t1b + 5 * t1c) FROM - (SELECT t1c as t1a, t1a as t1b, t0a as t1c - FROM t1 - WHERE t1a = t0a - UNION ALL - SELECT t0a as t2b, t2c as t1a, t0b as t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0 --- !query schema -struct --- !query output -1 32 -2 NULL - - --- !query -SELECT t0a, (SELECT count(t1c) FROM - (SELECT t1c - FROM t1 - WHERE t1a = t0a - UNION DISTINCT - SELECT t2c - FROM t2 - WHERE t2b = t0b) -) -FROM t0 --- !query schema -struct --- !query output -1 2 -2 0 - - --- !query -SELECT t0a, (SELECT sum(d) FROM - (SELECT t1a - t0a as d - FROM t1 - UNION ALL - SELECT t2a - t0a as d - FROM t2) -) -FROM t0 --- !query schema -struct --- !query output -1 1 -2 -2 - - --- !query -SELECT t0a, (SELECT sum(d) FROM - (SELECT sum(t0a) as d - FROM t1 - UNION ALL - SELECT sum(t2a) + t0a as d - FROM t2) -) -FROM t0 --- !query schema -struct<> --- !query output -org.apache.spark.sql.AnalysisException -{ - "errorClass" : "UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.CORRELATED_REFERENCE", - "sqlState" : "0A000", - "messageParameters" : { - "sqlExprs" : "\"sum(t0a) AS d\"" - }, - "queryContext" : [ { - "objectType" : "", - "objectName" : "", - "startIndex" : 36, - "stopIndex" : 67, - "fragment" : "SELECT sum(t0a) as d\n FROM t1" - } ] -} - - --- !query -select * -from -( - select t1.id c1, ( - select sum(c) - from ( - select t2.id * t2.id c - from range (1, 2) t2 where t1.id = t2.id - group by t2.id - ) - ) c2 - from range (1, 3) t1 -) t -where t.c2 is not null --- !query schema -struct --- !query output -1 1 diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala index 5067de74ea7e..345971e9ffc0 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala @@ -134,6 +134,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "subquery/negative-cases/invalid-correlation.sql", "subquery/negative-cases/subq-input-typecheck.sql", "subquery/scalar-subquery/scalar-subquery-predicate.sql", + "subquery/scalar-subquery/scalar-subquery-select.sql", "subquery/subquery-in-from.sql", "postgreSQL/aggregates_part1.sql", "postgreSQL/aggregates_part2.sql", @@ -241,9 +242,6 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "group-by.sql", "udf/udf-group-by.sql", // Overwrite some results of regr_intercept, regr_r2, corr. - "linear-regression.sql", - // Exception string doesn't match for - // SELECT (SELECT a FROM (SELECT 1 AS a UNION ALL SELECT 2 AS a) t) AS b - "subquery/scalar-subquery/scalar-subquery-select.sql" + "linear-regression.sql" ) } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 3a993189d6ae..505417aebb50 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -80,8 +80,6 @@ class VeloxTestSettings extends BackendTestSettings { // NEW SUITE: disable as it expects exception which doesn't happen when offloaded to gluten .exclude( "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") - // gluten throws different exception - .excludeByPrefix("SCALAR_SUBQUERY_TOO_MANY_ROWS:") // Doesn't support unhex with failOnError=true. .exclude("CONVERSION_INVALID_INPUT: to_binary conversion function hex") enableSuite[GlutenQueryParsingErrorsSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala index d9b661c8a4a7..8896541c29d2 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala @@ -24,15 +24,4 @@ class GlutenQueryExecutionErrorsSuite override protected def getResourceParquetFilePath(name: String): String = { getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name } - - testGluten( - "SCALAR_SUBQUERY_TOO_MANY_ROWS: " + - "More than one row returned by a subquery used as an expression") { - val exception = intercept[IllegalStateException] { - sql("select (select a from (select 1 as a union all select 2 as a) t) as b").collect() - } - assert( - exception.getMessage.contains("more than one row returned by a subquery" + - " used as an expression")) - } } diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql b/gluten-ut/spark35/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql deleted file mode 100644 index 741292d2c0fa..000000000000 --- a/gluten-ut/spark35/src/test/resources/sql-tests/inputs/subquery/scalar-subquery/scalar-subquery-select.sql +++ /dev/null @@ -1,257 +0,0 @@ --- A test suite for scalar subquery in SELECT clause - -create temporary view t1 as select * from values - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 00:00:00.000', date '2014-04-04'), - ('val1b', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1a', 16S, 12, 21L, float(15.0), 20D, 20E2, timestamp '2014-06-04 01:02:00.001', date '2014-06-04'), - ('val1a', 16S, 12, 10L, float(15.0), 20D, 20E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:02:00.001', date '2014-05-05'), - ('val1d', null, 16, 22L, float(17.0), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', null), - ('val1d', null, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-07-04 01:02:00.001', null), - ('val1e', 10S, null, 25L, float(17.0), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-09-04 01:02:00.001', date '2014-09-04'), - ('val1d', 10S, null, 12L, float(17.0), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:02:00.001', date '2014-04-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04') - as t1(t1a, t1b, t1c, t1d, t1e, t1f, t1g, t1h, t1i); - -create temporary view t2 as select * from values - ('val2a', 6S, 12, 14L, float(15), 20D, 20E2, timestamp '2014-04-04 01:01:00.000', date '2014-04-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 119L, float(17), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1c', 12S, 16, 219L, float(17), 25D, 26E2, timestamp '2016-05-04 01:01:00.000', date '2016-05-04'), - ('val1b', null, 16, 319L, float(17), 25D, 26E2, timestamp '2017-05-04 01:01:00.000', null), - ('val2e', 8S, null, 419L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1f', 19S, null, 519L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 12S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-05'), - ('val1e', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:01:00.000', date '2014-09-04'), - ('val1f', 19S, null, 19L, float(17), 25D, 26E2, timestamp '2014-10-04 01:01:00.000', date '2014-10-04'), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', null) - as t2(t2a, t2b, t2c, t2d, t2e, t2f, t2g, t2h, t2i); - -create temporary view t3 as select * from values - ('val3a', 6S, 12, 110L, float(15), 20D, 20E2, timestamp '2014-04-04 01:02:00.000', date '2014-04-04'), - ('val3a', 6S, 12, 10L, float(15), 20D, 20E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 219L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 319L, float(17), 25D, 26E2, timestamp '2014-06-04 01:02:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:02:00.000', date '2014-07-04'), - ('val3c', 17S, 16, 519L, float(17), 25D, 26E2, timestamp '2014-08-04 01:02:00.000', date '2014-08-04'), - ('val3c', 17S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:02:00.000', date '2014-09-05'), - ('val1b', null, 16, 419L, float(17), 25D, 26E2, timestamp '2014-10-04 01:02:00.000', null), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-11-04 01:02:00.000', null), - ('val3b', 8S, null, 719L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val3b', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2015-05-04 01:02:00.000', date '2015-05-04') - as t3(t3a, t3b, t3c, t3d, t3e, t3f, t3g, t3h, t3i); - --- Group 1: scalar subquery in SELECT clause --- no correlation --- TC 01.01 --- more than one scalar subquery -SELECT (SELECT min(t3d) FROM t3) min_t3d, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c'; - --- TC 01.02 --- scalar subquery in an IN subquery -SELECT t1a, count(*) -FROM t1 -WHERE t1c IN (SELECT (SELECT min(t3c) FROM t3) - FROM t2 - GROUP BY t2g - HAVING count(*) > 1) -GROUP BY t1a; - --- TC 01.03 --- under a set op -SELECT (SELECT min(t3d) FROM t3) min_t3d, - null -FROM t1 -WHERE t1a = 'val1c' -UNION -SELECT null, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c'; - --- TC 01.04 -SELECT (SELECT min(t3c) FROM t3) min_t3d -FROM t1 -WHERE t1a = 'val1a' -INTERSECT -SELECT (SELECT min(t2c) FROM t2) min_t2d -FROM t1 -WHERE t1a = 'val1d'; - --- TC 01.05 -SELECT q1.t1a, q2.t2a, q1.min_t3d, q2.avg_t3d -FROM (SELECT t1a, (SELECT min(t3d) FROM t3) min_t3d - FROM t1 - WHERE t1a IN ('val1e', 'val1c')) q1 - FULL OUTER JOIN - (SELECT t2a, (SELECT avg(t3d) FROM t3) avg_t3d - FROM t2 - WHERE t2a IN ('val1c', 'val2a')) q2 -ON q1.t1a = q2.t2a -AND q1.min_t3d < q2.avg_t3d; - --- Group 2: scalar subquery in SELECT clause --- with correlation --- TC 02.01 -SELECT (SELECT min(t3d) FROM t3 WHERE t3.t3a = t1.t1a) min_t3d, - (SELECT max(t2h) FROM t2 WHERE t2.t2a = t1.t1a) max_t2h -FROM t1 -WHERE t1a = 'val1b'; - --- TC 02.02 -SELECT (SELECT min(t3d) FROM t3 WHERE t3a = t1a) min_t3d -FROM t1 -WHERE t1a = 'val1b' -MINUS -SELECT (SELECT min(t3d) FROM t3) abs_min_t3d -FROM t1 -WHERE t1a = 'val1b'; - --- TC 02.03 -SELECT t1a, t1b -FROM t1 -WHERE NOT EXISTS (SELECT (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) dummy - FROM t3 - WHERE t3b < (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) - AND t3a = t1a); - --- SPARK-34876: Non-nullable aggregates should not return NULL in a correlated subquery -SELECT t1a, - (SELECT count(t2d) FROM t2 WHERE t2a = t1a) count_t2, - (SELECT count_if(t2d > 0) FROM t2 WHERE t2a = t1a) count_if_t2, - (SELECT approx_count_distinct(t2d) FROM t2 WHERE t2a = t1a) approx_count_distinct_t2, - (SELECT collect_list(t2d) FROM t2 WHERE t2a = t1a) collect_list_t2, - (SELECT sort_array(collect_set(t2d)) FROM t2 WHERE t2a = t1a) collect_set_t2, - (SELECT hex(count_min_sketch(t2d, 0.5d, 0.5d, 1)) FROM t2 WHERE t2a = t1a) collect_set_t2 -FROM t1; - --- SPARK-36028: Allow Project to host outer references in scalar subqueries -SELECT t1c, (SELECT t1c) FROM t1; -SELECT t1c, (SELECT t1c WHERE t1c = 8) FROM t1; -SELECT t1c, t1d, (SELECT c + d FROM (SELECT t1c AS c, t1d AS d)) FROM t1; -SELECT t1c, (SELECT SUM(c) FROM (SELECT t1c AS c)) FROM t1; -SELECT t1a, (SELECT SUM(t2b) FROM t2 JOIN (SELECT t1a AS a) ON t2a = a) FROM t1; - --- CTE in correlated scalar subqueries -CREATE OR REPLACE TEMPORARY VIEW t1 AS VALUES (0, 1), (1, 2) t1(c1, c2); -CREATE OR REPLACE TEMPORARY VIEW t2 AS VALUES (0, 2), (0, 3) t2(c1, c2); - --- Single row subquery -SELECT c1, (WITH t AS (SELECT 1 AS a) SELECT a + c1 FROM t) FROM t1; --- Correlation in CTE. -SELECT c1, (WITH t AS (SELECT * FROM t2 WHERE c1 = t1.c1) SELECT SUM(c2) FROM t) FROM t1; --- Multiple CTE definitions. -SELECT c1, ( - WITH t3 AS (SELECT c1 + 1 AS c1, c2 + 1 AS c2 FROM t2), - t4 AS (SELECT * FROM t3 WHERE t1.c1 = c1) - SELECT SUM(c2) FROM t4 -) FROM t1; --- Multiple CTE references. -SELECT c1, ( - WITH t AS (SELECT * FROM t2) - SELECT SUM(c2) FROM (SELECT c1, c2 FROM t UNION SELECT c2, c1 FROM t) r(c1, c2) - WHERE c1 = t1.c1 -) FROM t1; --- Reference CTE in both the main query and the subquery. -WITH v AS (SELECT * FROM t2) -SELECT * FROM t1 WHERE c1 > ( - WITH t AS (SELECT * FROM t2) - SELECT COUNT(*) FROM v WHERE c1 = t1.c1 AND c1 > (SELECT SUM(c2) FROM t WHERE c1 = v.c1) -); --- Single row subquery that references CTE in the main query. -WITH t AS (SELECT 1 AS a) -SELECT c1, (SELECT a FROM t WHERE a = c1) FROM t1; --- Multiple CTE references with non-deterministic CTEs. -WITH -v1 AS (SELECT c1, c2, rand(0) c3 FROM t1), -v2 AS (SELECT c1, c2, rand(0) c4 FROM v1 WHERE c3 IN (SELECT c3 FROM v1)) -SELECT c1, ( - WITH v3 AS (SELECT c1, c2, rand(0) c5 FROM t2) - SELECT COUNT(*) FROM ( - SELECT * FROM v2 WHERE c1 > 0 - UNION SELECT * FROM v2 WHERE c2 > 0 - UNION SELECT * FROM v3 WHERE c2 > 0 - ) WHERE c1 = v1.c1 -) FROM v1; - --- Multi-value subquery error -SELECT (SELECT a FROM (SELECT 1 AS a UNION ALL SELECT 2 AS a) t) AS b; - --- SPARK-36114: Support correlated non-equality predicates -CREATE OR REPLACE TEMP VIEW t1(c1, c2) AS (VALUES (0, 1), (1, 2)); -CREATE OR REPLACE TEMP VIEW t2(c1, c2) AS (VALUES (0, 2), (0, 3)); - --- Neumann example Q2 -CREATE OR REPLACE TEMP VIEW students(id, name, major, year) AS (VALUES - (0, 'A', 'CS', 2022), - (1, 'B', 'CS', 2022), - (2, 'C', 'Math', 2022)); -CREATE OR REPLACE TEMP VIEW exams(sid, course, curriculum, grade, date) AS (VALUES - (0, 'C1', 'CS', 4, 2020), - (0, 'C2', 'CS', 3, 2021), - (1, 'C1', 'CS', 2, 2020), - (1, 'C2', 'CS', 1, 2021)); - -SELECT students.name, exams.course -FROM students, exams -WHERE students.id = exams.sid - AND (students.major = 'CS' OR students.major = 'Games Eng') - AND exams.grade >= ( - SELECT avg(exams.grade) + 1 - FROM exams - WHERE students.id = exams.sid - OR (exams.curriculum = students.major AND students.year > exams.date)); - --- Correlated non-equality predicates -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 > t2.c1) FROM t1; -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 >= t2.c1 AND t1.c2 < t2.c2) FROM t1; - --- Correlated non-equality predicates with the COUNT bug. -SELECT (SELECT count(*) FROM t2 WHERE t1.c1 > t2.c1) FROM t1; - --- Correlated equality predicates that are not supported after SPARK-35080 -SELECT c, ( - SELECT count(*) - FROM (VALUES ('ab'), ('abc'), ('bc')) t2(c) - WHERE t1.c = substring(t2.c, 1, 1) -) FROM (VALUES ('a'), ('b')) t1(c); - -SELECT c, ( - SELECT count(*) - FROM (VALUES (0, 6), (1, 5), (2, 4), (3, 3)) t1(a, b) - WHERE a + b = c -) FROM (VALUES (6)) t2(c); - --- SPARK-43156: scalar subquery with Literal result like `COUNT(1) is null` -SELECT *, (SELECT count(1) is null FROM t2 WHERE t1.c1 = t2.c1) FROM t1; - -select (select f from (select false as f, max(c2) from t1 where t1.c1 = t1.c1)) from t2; - --- SPARK-43596: handle IsNull when rewriting the domain join -set spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline=false; -WITH T AS (SELECT 1 AS a) -SELECT (SELECT sum(1) FROM T WHERE a = col OR upper(col)= 'Y') -FROM (SELECT null as col) as foo; -set spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline=true; - --- SPARK-43760: the result of the subquery can be NULL. -select * from ( - select t1.id c1, ( - select t2.id c from range (1, 2) t2 - where t1.id = t2.id ) c2 - from range (1, 3) t1 ) t -where t.c2 is not null; diff --git a/gluten-ut/spark35/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out b/gluten-ut/spark35/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out deleted file mode 100644 index 5c6f141d8505..000000000000 --- a/gluten-ut/spark35/src/test/resources/sql-tests/results/subquery/scalar-subquery/scalar-subquery-select.sql.out +++ /dev/null @@ -1,614 +0,0 @@ --- Automatically generated by GlutenSQLQueryTestSuite --- Number of queries: 52 - - --- !query -create temporary view t1 as select * from values - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 00:00:00.000', date '2014-04-04'), - ('val1b', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1a', 16S, 12, 21L, float(15.0), 20D, 20E2, timestamp '2014-06-04 01:02:00.001', date '2014-06-04'), - ('val1a', 16S, 12, 10L, float(15.0), 20D, 20E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:02:00.001', date '2014-05-05'), - ('val1d', null, 16, 22L, float(17.0), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', null), - ('val1d', null, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-07-04 01:02:00.001', null), - ('val1e', 10S, null, 25L, float(17.0), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-09-04 01:02:00.001', date '2014-09-04'), - ('val1d', 10S, null, 12L, float(17.0), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1a', 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:02:00.001', date '2014-04-04'), - ('val1e', 10S, null, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04') - as t1(t1a, t1b, t1c, t1d, t1e, t1f, t1g, t1h, t1i) --- !query schema -struct<> --- !query output - - - --- !query -create temporary view t2 as select * from values - ('val2a', 6S, 12, 14L, float(15), 20D, 20E2, timestamp '2014-04-04 01:01:00.000', date '2014-04-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 119L, float(17), 25D, 26E2, timestamp '2015-05-04 01:01:00.000', date '2015-05-04'), - ('val1c', 12S, 16, 219L, float(17), 25D, 26E2, timestamp '2016-05-04 01:01:00.000', date '2016-05-04'), - ('val1b', null, 16, 319L, float(17), 25D, 26E2, timestamp '2017-05-04 01:01:00.000', null), - ('val2e', 8S, null, 419L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1f', 19S, null, 519L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-06-04 01:01:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:01:00.000', date '2014-07-04'), - ('val1c', 12S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-08-04 01:01:00.000', date '2014-08-05'), - ('val1e', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:01:00.000', date '2014-09-04'), - ('val1f', 19S, null, 19L, float(17), 25D, 26E2, timestamp '2014-10-04 01:01:00.000', date '2014-10-04'), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', null) - as t2(t2a, t2b, t2c, t2d, t2e, t2f, t2g, t2h, t2i) --- !query schema -struct<> --- !query output - - - --- !query -create temporary view t3 as select * from values - ('val3a', 6S, 12, 110L, float(15), 20D, 20E2, timestamp '2014-04-04 01:02:00.000', date '2014-04-04'), - ('val3a', 6S, 12, 10L, float(15), 20D, 20E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 219L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 10S, 12, 19L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val1b', 8S, 16, 319L, float(17), 25D, 26E2, timestamp '2014-06-04 01:02:00.000', date '2014-06-04'), - ('val1b', 8S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-07-04 01:02:00.000', date '2014-07-04'), - ('val3c', 17S, 16, 519L, float(17), 25D, 26E2, timestamp '2014-08-04 01:02:00.000', date '2014-08-04'), - ('val3c', 17S, 16, 19L, float(17), 25D, 26E2, timestamp '2014-09-04 01:02:00.000', date '2014-09-05'), - ('val1b', null, 16, 419L, float(17), 25D, 26E2, timestamp '2014-10-04 01:02:00.000', null), - ('val1b', null, 16, 19L, float(17), 25D, 26E2, timestamp '2014-11-04 01:02:00.000', null), - ('val3b', 8S, null, 719L, float(17), 25D, 26E2, timestamp '2014-05-04 01:02:00.000', date '2014-05-04'), - ('val3b', 8S, null, 19L, float(17), 25D, 26E2, timestamp '2015-05-04 01:02:00.000', date '2015-05-04') - as t3(t3a, t3b, t3c, t3d, t3e, t3f, t3g, t3h, t3i) --- !query schema -struct<> --- !query output - - - --- !query -SELECT (SELECT min(t3d) FROM t3) min_t3d, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c' --- !query schema -struct --- !query output -10 2017-05-04 01:01:00 - - --- !query -SELECT t1a, count(*) -FROM t1 -WHERE t1c IN (SELECT (SELECT min(t3c) FROM t3) - FROM t2 - GROUP BY t2g - HAVING count(*) > 1) -GROUP BY t1a --- !query schema -struct --- !query output -val1a 2 - - --- !query -SELECT (SELECT min(t3d) FROM t3) min_t3d, - null -FROM t1 -WHERE t1a = 'val1c' -UNION -SELECT null, - (SELECT max(t2h) FROM t2) max_t2h -FROM t1 -WHERE t1a = 'val1c' --- !query schema -struct --- !query output -10 NULL -NULL 2017-05-04 01:01:00 - - --- !query -SELECT (SELECT min(t3c) FROM t3) min_t3d -FROM t1 -WHERE t1a = 'val1a' -INTERSECT -SELECT (SELECT min(t2c) FROM t2) min_t2d -FROM t1 -WHERE t1a = 'val1d' --- !query schema -struct --- !query output -12 - - --- !query -SELECT q1.t1a, q2.t2a, q1.min_t3d, q2.avg_t3d -FROM (SELECT t1a, (SELECT min(t3d) FROM t3) min_t3d - FROM t1 - WHERE t1a IN ('val1e', 'val1c')) q1 - FULL OUTER JOIN - (SELECT t2a, (SELECT avg(t3d) FROM t3) avg_t3d - FROM t2 - WHERE t2a IN ('val1c', 'val2a')) q2 -ON q1.t1a = q2.t2a -AND q1.min_t3d < q2.avg_t3d --- !query schema -struct --- !query output -NULL val2a NULL 200.83333333333334 -val1c val1c 10 200.83333333333334 -val1c val1c 10 200.83333333333334 -val1e NULL 10 NULL -val1e NULL 10 NULL -val1e NULL 10 NULL - - --- !query -SELECT (SELECT min(t3d) FROM t3 WHERE t3.t3a = t1.t1a) min_t3d, - (SELECT max(t2h) FROM t2 WHERE t2.t2a = t1.t1a) max_t2h -FROM t1 -WHERE t1a = 'val1b' --- !query schema -struct --- !query output -19 2017-05-04 01:01:00 - - --- !query -SELECT (SELECT min(t3d) FROM t3 WHERE t3a = t1a) min_t3d -FROM t1 -WHERE t1a = 'val1b' -MINUS -SELECT (SELECT min(t3d) FROM t3) abs_min_t3d -FROM t1 -WHERE t1a = 'val1b' --- !query schema -struct --- !query output -19 - - --- !query -SELECT t1a, t1b -FROM t1 -WHERE NOT EXISTS (SELECT (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) dummy - FROM t3 - WHERE t3b < (SELECT max(t2b) - FROM t2 LEFT JOIN t1 - ON t2a = t1a - WHERE t2c = t3c) - AND t3a = t1a) --- !query schema -struct --- !query output -val1a 16 -val1a 16 -val1a 6 -val1a 6 -val1c 8 -val1d 10 -val1d NULL -val1d NULL -val1e 10 -val1e 10 -val1e 10 - - --- !query -SELECT t1a, - (SELECT count(t2d) FROM t2 WHERE t2a = t1a) count_t2, - (SELECT count_if(t2d > 0) FROM t2 WHERE t2a = t1a) count_if_t2, - (SELECT approx_count_distinct(t2d) FROM t2 WHERE t2a = t1a) approx_count_distinct_t2, - (SELECT collect_list(t2d) FROM t2 WHERE t2a = t1a) collect_list_t2, - (SELECT sort_array(collect_set(t2d)) FROM t2 WHERE t2a = t1a) collect_set_t2, - (SELECT hex(count_min_sketch(t2d, 0.5d, 0.5d, 1)) FROM t2 WHERE t2a = t1a) collect_set_t2 -FROM t1 --- !query schema -struct,collect_set_t2:array,collect_set_t2:string> --- !query output -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1a 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1b 6 6 3 [19,119,319,19,19,19] [19,119,319] 0000000100000000000000060000000100000004000000005D8D6AB90000000000000000000000000000000400000000000000010000000000000001 -val1c 2 2 2 [219,19] [19,219] 0000000100000000000000020000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000001 -val1d 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1d 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1d 0 0 0 [] [] 0000000100000000000000000000000100000004000000005D8D6AB90000000000000000000000000000000000000000000000000000000000000000 -val1e 1 1 1 [19] [19] 0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000 -val1e 1 1 1 [19] [19] 0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000 -val1e 1 1 1 [19] [19] 0000000100000000000000010000000100000004000000005D8D6AB90000000000000000000000000000000100000000000000000000000000000000 - - --- !query -SELECT t1c, (SELECT t1c) FROM t1 --- !query schema -struct --- !query output -12 12 -12 12 -16 16 -16 16 -16 16 -16 16 -8 8 -8 8 -NULL NULL -NULL NULL -NULL NULL -NULL NULL - - --- !query -SELECT t1c, (SELECT t1c WHERE t1c = 8) FROM t1 --- !query schema -struct --- !query output -12 NULL -12 NULL -16 NULL -16 NULL -16 NULL -16 NULL -8 8 -8 8 -NULL NULL -NULL NULL -NULL NULL -NULL NULL - - --- !query -SELECT t1c, t1d, (SELECT c + d FROM (SELECT t1c AS c, t1d AS d)) FROM t1 --- !query schema -struct --- !query output -12 10 22 -12 21 33 -16 19 35 -16 19 35 -16 19 35 -16 22 38 -8 10 18 -8 10 18 -NULL 12 NULL -NULL 19 NULL -NULL 19 NULL -NULL 25 NULL - - --- !query -SELECT t1c, (SELECT SUM(c) FROM (SELECT t1c AS c)) FROM t1 --- !query schema -struct --- !query output -12 12 -12 12 -16 16 -16 16 -16 16 -16 16 -8 8 -8 8 -NULL NULL -NULL NULL -NULL NULL -NULL NULL - - --- !query -SELECT t1a, (SELECT SUM(t2b) FROM t2 JOIN (SELECT t1a AS a) ON t2a = a) FROM t1 --- !query schema -struct --- !query output -val1a NULL -val1a NULL -val1a NULL -val1a NULL -val1b 36 -val1c 24 -val1d NULL -val1d NULL -val1d NULL -val1e 8 -val1e 8 -val1e 8 - - --- !query -CREATE OR REPLACE TEMPORARY VIEW t1 AS VALUES (0, 1), (1, 2) t1(c1, c2) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMPORARY VIEW t2 AS VALUES (0, 2), (0, 3) t2(c1, c2) --- !query schema -struct<> --- !query output - - - --- !query -SELECT c1, (WITH t AS (SELECT 1 AS a) SELECT a + c1 FROM t) FROM t1 --- !query schema -struct --- !query output -0 1 -1 2 - - --- !query -SELECT c1, (WITH t AS (SELECT * FROM t2 WHERE c1 = t1.c1) SELECT SUM(c2) FROM t) FROM t1 --- !query schema -struct --- !query output -0 5 -1 NULL - - --- !query -SELECT c1, ( - WITH t3 AS (SELECT c1 + 1 AS c1, c2 + 1 AS c2 FROM t2), - t4 AS (SELECT * FROM t3 WHERE t1.c1 = c1) - SELECT SUM(c2) FROM t4 -) FROM t1 --- !query schema -struct --- !query output -0 NULL -1 7 - - --- !query -SELECT c1, ( - WITH t AS (SELECT * FROM t2) - SELECT SUM(c2) FROM (SELECT c1, c2 FROM t UNION SELECT c2, c1 FROM t) r(c1, c2) - WHERE c1 = t1.c1 -) FROM t1 --- !query schema -struct --- !query output -0 5 -1 NULL - - --- !query -WITH v AS (SELECT * FROM t2) -SELECT * FROM t1 WHERE c1 > ( - WITH t AS (SELECT * FROM t2) - SELECT COUNT(*) FROM v WHERE c1 = t1.c1 AND c1 > (SELECT SUM(c2) FROM t WHERE c1 = v.c1) -) --- !query schema -struct --- !query output -1 2 - - --- !query -WITH t AS (SELECT 1 AS a) -SELECT c1, (SELECT a FROM t WHERE a = c1) FROM t1 --- !query schema -struct --- !query output -0 NULL -1 1 - - --- !query -WITH -v1 AS (SELECT c1, c2, rand(0) c3 FROM t1), -v2 AS (SELECT c1, c2, rand(0) c4 FROM v1 WHERE c3 IN (SELECT c3 FROM v1)) -SELECT c1, ( - WITH v3 AS (SELECT c1, c2, rand(0) c5 FROM t2) - SELECT COUNT(*) FROM ( - SELECT * FROM v2 WHERE c1 > 0 - UNION SELECT * FROM v2 WHERE c2 > 0 - UNION SELECT * FROM v3 WHERE c2 > 0 - ) WHERE c1 = v1.c1 -) FROM v1 --- !query schema -struct --- !query output -0 3 -1 1 - - --- !query -SELECT (SELECT a FROM (SELECT 1 AS a UNION ALL SELECT 2 AS a) t) AS b --- !query schema -struct<> --- !query output -java.lang.IllegalStateException -more than one row returned by a subquery used as an expression: -Subquery subquery#1, [id=#2] -+- AdaptiveSparkPlan isFinalPlan=true - +- == Final Plan == - VeloxColumnarToRowExec - +- ColumnarUnion - :- ProjectExecTransformer [1 AS a#3] - : +- InputIteratorTransformer[fake_column#4] - : +- InputAdapter - : +- RowToVeloxColumnar - : +- Scan OneRowRelation[fake_column#4] - +- ProjectExecTransformer [2 AS a#5] - +- InputIteratorTransformer[fake_column#6] - +- InputAdapter - +- RowToVeloxColumnar - +- Scan OneRowRelation[fake_column#6] - +- == Initial Plan == - Union - :- Project [1 AS a#3] - : +- Scan OneRowRelation[] - +- Project [2 AS a#5] - +- Scan OneRowRelation[] - - --- !query -CREATE OR REPLACE TEMP VIEW t1(c1, c2) AS (VALUES (0, 1), (1, 2)) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW t2(c1, c2) AS (VALUES (0, 2), (0, 3)) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW students(id, name, major, year) AS (VALUES - (0, 'A', 'CS', 2022), - (1, 'B', 'CS', 2022), - (2, 'C', 'Math', 2022)) --- !query schema -struct<> --- !query output - - - --- !query -CREATE OR REPLACE TEMP VIEW exams(sid, course, curriculum, grade, date) AS (VALUES - (0, 'C1', 'CS', 4, 2020), - (0, 'C2', 'CS', 3, 2021), - (1, 'C1', 'CS', 2, 2020), - (1, 'C2', 'CS', 1, 2021)) --- !query schema -struct<> --- !query output - - - --- !query -SELECT students.name, exams.course -FROM students, exams -WHERE students.id = exams.sid - AND (students.major = 'CS' OR students.major = 'Games Eng') - AND exams.grade >= ( - SELECT avg(exams.grade) + 1 - FROM exams - WHERE students.id = exams.sid - OR (exams.curriculum = students.major AND students.year > exams.date)) --- !query schema -struct --- !query output -A C1 - - --- !query -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 > t2.c1) FROM t1 --- !query schema -struct --- !query output -2 -NULL - - --- !query -SELECT (SELECT min(c2) FROM t2 WHERE t1.c1 >= t2.c1 AND t1.c2 < t2.c2) FROM t1 --- !query schema -struct --- !query output -2 -3 - - --- !query -SELECT (SELECT count(*) FROM t2 WHERE t1.c1 > t2.c1) FROM t1 --- !query schema -struct --- !query output -0 -2 - - --- !query -SELECT c, ( - SELECT count(*) - FROM (VALUES ('ab'), ('abc'), ('bc')) t2(c) - WHERE t1.c = substring(t2.c, 1, 1) -) FROM (VALUES ('a'), ('b')) t1(c) --- !query schema -struct --- !query output -a 2 -b 1 - - --- !query -SELECT c, ( - SELECT count(*) - FROM (VALUES (0, 6), (1, 5), (2, 4), (3, 3)) t1(a, b) - WHERE a + b = c -) FROM (VALUES (6)) t2(c) --- !query schema -struct --- !query output -6 4 - - --- !query -SELECT *, (SELECT count(1) is null FROM t2 WHERE t1.c1 = t2.c1) FROM t1 --- !query schema -struct --- !query output -0 1 false -1 2 false - - --- !query -select (select f from (select false as f, max(c2) from t1 where t1.c1 = t1.c1)) from t2 --- !query schema -struct --- !query output -false -false - - --- !query -set spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline=false --- !query schema -struct --- !query output -spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline false - - --- !query -WITH T AS (SELECT 1 AS a) -SELECT (SELECT sum(1) FROM T WHERE a = col OR upper(col)= 'Y') -FROM (SELECT null as col) as foo --- !query schema -struct --- !query output -NULL - - --- !query -set spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline=true --- !query schema -struct --- !query output -spark.sql.optimizer.optimizeOneRowRelationSubquery.alwaysInline true - - --- !query -select * from ( - select t1.id c1, ( - select t2.id c from range (1, 2) t2 - where t1.id = t2.id ) c2 - from range (1, 3) t1 ) t -where t.c2 is not null --- !query schema -struct --- !query output -1 1 diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala index 5067de74ea7e..345971e9ffc0 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala @@ -134,6 +134,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "subquery/negative-cases/invalid-correlation.sql", "subquery/negative-cases/subq-input-typecheck.sql", "subquery/scalar-subquery/scalar-subquery-predicate.sql", + "subquery/scalar-subquery/scalar-subquery-select.sql", "subquery/subquery-in-from.sql", "postgreSQL/aggregates_part1.sql", "postgreSQL/aggregates_part2.sql", @@ -241,9 +242,6 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "group-by.sql", "udf/udf-group-by.sql", // Overwrite some results of regr_intercept, regr_r2, corr. - "linear-regression.sql", - // Exception string doesn't match for - // SELECT (SELECT a FROM (SELECT 1 AS a UNION ALL SELECT 2 AS a) t) AS b - "subquery/scalar-subquery/scalar-subquery-select.sql" + "linear-regression.sql" ) } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 98942462aec8..27557f92046d 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -81,8 +81,6 @@ class VeloxTestSettings extends BackendTestSettings { // NEW SUITE: disable as it expects exception which doesn't happen when offloaded to gluten .exclude( "INCONSISTENT_BEHAVIOR_CROSS_VERSION: compatibility with Spark 2.4/3.2 in reading/writing dates") - // gluten throws different exception - .excludeByPrefix("SCALAR_SUBQUERY_TOO_MANY_ROWS:") // Doesn't support unhex with failOnError=true. .exclude("CONVERSION_INVALID_INPUT: to_binary conversion function hex") enableSuite[GlutenQueryParsingErrorsSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala index d9b661c8a4a7..8896541c29d2 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala @@ -24,15 +24,4 @@ class GlutenQueryExecutionErrorsSuite override protected def getResourceParquetFilePath(name: String): String = { getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name } - - testGluten( - "SCALAR_SUBQUERY_TOO_MANY_ROWS: " + - "More than one row returned by a subquery used as an expression") { - val exception = intercept[IllegalStateException] { - sql("select (select a from (select 1 as a union all select 2 as a) t) as b").collect() - } - assert( - exception.getMessage.contains("more than one row returned by a subquery" + - " used as an expression")) - } } From 63db789311a2ee65500b3011592e62cc026f04e8 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Tue, 28 May 2024 16:08:16 +0800 Subject: [PATCH 162/402] [VL] Fix build error (#5891) --- .../org/apache/gluten/execution/SampleExecTransformer.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala index 86189392af75..6f9ef34282bf 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SampleExecTransformer.scala @@ -112,8 +112,8 @@ case class SampleExecTransformer( doNativeValidation(substraitContext, relNode) } - override def doTransform(context: SubstraitContext): TransformContext = { - val childCtx = child.asInstanceOf[TransformSupport].doTransform(context) + override protected def doTransform(context: SubstraitContext): TransformContext = { + val childCtx = child.asInstanceOf[TransformSupport].transform(context) val operatorId = context.nextOperatorId(this.nodeName) val currRel = getRelNode(context, condition, child.output, operatorId, childCtx.root, validation = false) From c20437a89e11896962ba13f3495d5369e9321759 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 28 May 2024 19:53:49 +0800 Subject: [PATCH 163/402] [VL] Upgrade folly to v2024.04.01.00 (#5314) --- dev/vcpkg/ports/folly/boost-1.70.patch | 4 +- .../disable-non-underscore-posix-names.patch | 34 ---- ...able-uninitialized-resize-on-new-stl.patch | 34 ++++ dev/vcpkg/ports/folly/fix-deps.patch | 152 ++++++++++-------- .../ports/folly/fix-unistd-include.patch | 13 ++ dev/vcpkg/ports/folly/portfile.cmake | 20 +-- .../ports/folly/reorder-glog-gflags.patch | 28 ---- .../ports/folly/vcpkg-cmake-wrapper.cmake | 10 +- dev/vcpkg/ports/folly/vcpkg.json | 10 +- 9 files changed, 152 insertions(+), 153 deletions(-) delete mode 100644 dev/vcpkg/ports/folly/disable-non-underscore-posix-names.patch create mode 100644 dev/vcpkg/ports/folly/disable-uninitialized-resize-on-new-stl.patch create mode 100644 dev/vcpkg/ports/folly/fix-unistd-include.patch delete mode 100644 dev/vcpkg/ports/folly/reorder-glog-gflags.patch diff --git a/dev/vcpkg/ports/folly/boost-1.70.patch b/dev/vcpkg/ports/folly/boost-1.70.patch index 298516f826b0..da6b81788fc8 100644 --- a/dev/vcpkg/ports/folly/boost-1.70.patch +++ b/dev/vcpkg/ports/folly/boost-1.70.patch @@ -1,5 +1,5 @@ diff --git a/folly/portability/PThread.cpp b/folly/portability/PThread.cpp -index 2126ec0..ef82ade 100644 +index 777ac21ef..a4fc13a09 100644 --- a/folly/portability/PThread.cpp +++ b/folly/portability/PThread.cpp @@ -20,6 +20,8 @@ @@ -11,7 +11,7 @@ index 2126ec0..ef82ade 100644 #include -@@ -691,7 +693,7 @@ int pthread_setspecific(pthread_key_t key, const void* value) { +@@ -682,7 +684,7 @@ int pthread_setspecific(pthread_key_t key, const void* value) { boost::detail::thread::cleanup_caller_t(), boost::detail::thread::cleanup_func_t(), #else diff --git a/dev/vcpkg/ports/folly/disable-non-underscore-posix-names.patch b/dev/vcpkg/ports/folly/disable-non-underscore-posix-names.patch deleted file mode 100644 index c039947920b1..000000000000 --- a/dev/vcpkg/ports/folly/disable-non-underscore-posix-names.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/folly/portability/Windows.h b/folly/portability/Windows.h -index 86fd0f9..ccad11e 100644 ---- a/folly/portability/Windows.h -+++ b/folly/portability/Windows.h -@@ -32,24 +32,17 @@ - // disabled to ensure all of the normal names get declared properly. - #include - --#ifndef __STDC__ --/* nolint */ --#define __STDC__ 1 - #pragma push_macro("_CRT_DECLARE_NONSTDC_NAMES") - #ifdef _CRT_DECLARE_NONSTDC_NAMES - #undef _CRT_DECLARE_NONSTDC_NAMES - #endif -+ -+#include - #pragma push_macro("_CRT_INTERNAL_NONSTDC_NAMES") --#undef _CRT_INTERNAL_NONSTDC_NAMES --#include // @manual nolint --#include // @manual nolint --#undef __STDC__ -+#define _CRT_INTERNAL_NONSTDC_NAMES 0 -+#include -+#include - #pragma pop_macro("_CRT_INTERNAL_NONSTDC_NAMES") --#pragma pop_macro("_CRT_DECLARE_NONSTDC_NAMES") --#else --#include // @manual nolint --#include // @manual nolint --#endif - - #if defined(min) || defined(max) - #error Windows.h needs to be included by this header, or else NOMINMAX needs \ diff --git a/dev/vcpkg/ports/folly/disable-uninitialized-resize-on-new-stl.patch b/dev/vcpkg/ports/folly/disable-uninitialized-resize-on-new-stl.patch new file mode 100644 index 000000000000..6e7b6b571823 --- /dev/null +++ b/dev/vcpkg/ports/folly/disable-uninitialized-resize-on-new-stl.patch @@ -0,0 +1,34 @@ +diff --git a/folly/memory/UninitializedMemoryHacks.h b/folly/memory/UninitializedMemoryHacks.h +index daf5eb735..1ac44d6b2 100644 +--- a/folly/memory/UninitializedMemoryHacks.h ++++ b/folly/memory/UninitializedMemoryHacks.h +@@ -101,6 +101,9 @@ template < + typename std::enable_if::value>::type> + inline void resizeWithoutInitialization( + std::basic_string& s, std::size_t n) { ++#if defined(_MSVC_STL_UPDATE) && _MSVC_STL_UPDATE >= 202206L ++ s.resize(n); ++#else + if (n <= s.size()) { + s.resize(n); + } else { +@@ -111,6 +114,7 @@ inline void resizeWithoutInitialization( + } + detail::unsafeStringSetLargerSize(s, n); + } ++#endif // defined(_MSVC_STL_UPDATE) && _MSVC_STL_UPDATE >= 202206L + } + + /** +@@ -278,8 +282,11 @@ struct MakeUnsafeStringSetLargerSize { + } // namespace folly + + #if defined(FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT) ++#if defined(_MSVC_STL_UPDATE) && _MSVC_STL_UPDATE >= 202206L ++#else + FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(char) + FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(wchar_t) ++#endif // defined(_MSVC_STL_UPDATE) && _MSVC_STL_UPDATE >= 202206L + #endif + + namespace folly { diff --git a/dev/vcpkg/ports/folly/fix-deps.patch b/dev/vcpkg/ports/folly/fix-deps.patch index 179fa71be73e..442ad8063686 100644 --- a/dev/vcpkg/ports/folly/fix-deps.patch +++ b/dev/vcpkg/ports/folly/fix-deps.patch @@ -1,12 +1,13 @@ diff --git a/CMake/folly-config.cmake.in b/CMake/folly-config.cmake.in -index 0b96f0a10..c90110287 100644 +index 0b96f0a10..1559f9f70 100644 --- a/CMake/folly-config.cmake.in +++ b/CMake/folly-config.cmake.in -@@ -29,10 +29,35 @@ endif() +@@ -29,10 +29,30 @@ endif() set(FOLLY_LIBRARIES Folly::folly) # Find folly's dependencies -find_dependency(fmt) ++find_dependency(fmt CONFIG) +find_dependency(double-conversion CONFIG) +find_dependency(glog CONFIG) +find_dependency(gflags CONFIG) @@ -27,12 +28,6 @@ index 0b96f0a10..c90110287 100644 +if (NOT @CMAKE_DISABLE_FIND_PACKAGE_LZ4@) + find_dependency(lz4 CONFIG) +endif() -+ -+if (@WITH_liburing@) -+ find_dependency(LibUring) -+endif() -+ -+find_dependency(fmt CONFIG) set(Boost_USE_STATIC_LIBS "@FOLLY_BOOST_LINK_STATIC@") -find_dependency(Boost 1.51.0 MODULE @@ -41,7 +36,7 @@ index 0b96f0a10..c90110287 100644 context filesystem diff --git a/CMake/folly-deps.cmake b/CMake/folly-deps.cmake -index 4b78e9f02..eb77e29c9 100644 +index 41a513a30..d75908f77 100644 --- a/CMake/folly-deps.cmake +++ b/CMake/folly-deps.cmake @@ -35,7 +35,7 @@ else() @@ -53,27 +48,17 @@ index 4b78e9f02..eb77e29c9 100644 COMPONENTS context filesystem -@@ -45,37 +45,38 @@ find_package(Boost 1.51.0 MODULE +@@ -45,37 +45,37 @@ find_package(Boost 1.51.0 MODULE thread REQUIRED ) -+set (Boost_LIBRARIES Boost::context Boost::filesystem Boost::program_options Boost::regex Boost::system Boost::thread -+ ) ++set(Boost_LIBRARIES Boost::boost Boost::context Boost::filesystem Boost::program_options Boost::regex Boost::system Boost::thread) list(APPEND FOLLY_LINK_LIBRARIES ${Boost_LIBRARIES}) -list(APPEND FOLLY_INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS}) - +- -find_package(DoubleConversion MODULE REQUIRED) -list(APPEND FOLLY_LINK_LIBRARIES ${DOUBLE_CONVERSION_LIBRARY}) -list(APPEND FOLLY_INCLUDE_DIRECTORIES ${DOUBLE_CONVERSION_INCLUDE_DIR}) -+find_package(double-conversion CONFIG REQUIRED) -+list(APPEND FOLLY_LINK_LIBRARIES double-conversion::double-conversion) - --find_package(Glog CONFIG REQUIRED) --set(FOLLY_HAVE_LIBGLOG ${GLOG_FOUND}) -+find_package(glog CONFIG REQUIRED) -+set(FOLLY_HAVE_LIBGLOG 1) - list(APPEND FOLLY_LINK_LIBRARIES glog::glog) --list(APPEND FOLLY_INCLUDE_DIRECTORIES ${GLOG_INCLUDE_DIR}) - -find_package(Gflags MODULE) -set(FOLLY_HAVE_LIBGFLAGS ${LIBGFLAGS_FOUND}) @@ -82,22 +67,21 @@ index 4b78e9f02..eb77e29c9 100644 - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBGFLAGS_INCLUDE_DIR}) - set(FOLLY_LIBGFLAGS_LIBRARY ${LIBGFLAGS_LIBRARY}) - set(FOLLY_LIBGFLAGS_INCLUDE ${LIBGFLAGS_INCLUDE_DIR}) -+ -+find_package(gflags CONFIG) -+if(TARGET gflags::gflags) -+ set(FOLLY_HAVE_LIBGFLAGS 1) -+ list(APPEND FOLLY_LINK_LIBRARIES gflags::gflags) -+ set(FOLLY_LIBGFLAGS_LIBRARY gflags::gflags) - endif() +-endif() + +-find_package(Glog MODULE) +-set(FOLLY_HAVE_LIBGLOG ${GLOG_FOUND}) +-list(APPEND FOLLY_LINK_LIBRARIES ${GLOG_LIBRARY}) +-list(APPEND FOLLY_INCLUDE_DIRECTORIES ${GLOG_INCLUDE_DIR}) ++find_package(double-conversion CONFIG REQUIRED) ++list(APPEND FOLLY_LINK_LIBRARIES double-conversion::double-conversion) -find_package(LibEvent MODULE REQUIRED) -list(APPEND FOLLY_LINK_LIBRARIES ${LIBEVENT_LIB}) -list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBEVENT_INCLUDE_DIR}) -+find_package(Libevent CONFIG REQUIRED) -+list(APPEND FOLLY_LINK_LIBRARIES libevent::core libevent::extra) -+if(NOT WIN32) -+ list(APPEND FOLLY_LINK_LIBRARIES libevent::pthreads) -+endif() ++find_package(glog CONFIG REQUIRED) ++set(FOLLY_HAVE_LIBGLOG 1) ++list(APPEND FOLLY_LINK_LIBRARIES glog::glog) -find_package(ZLIB MODULE) -set(FOLLY_HAVE_LIBZ ${ZLIB_FOUND}) @@ -105,18 +89,31 @@ index 4b78e9f02..eb77e29c9 100644 - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) - list(APPEND FOLLY_LINK_LIBRARIES ${ZLIB_LIBRARIES}) - list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES}) ++find_package(gflags CONFIG) ++if(TARGET gflags::gflags) ++ set(FOLLY_HAVE_LIBGFLAGS 1) ++ list(APPEND FOLLY_LINK_LIBRARIES gflags::gflags) ++ set(FOLLY_LIBGFLAGS_LIBRARY gflags::gflags) ++endif() ++ ++find_package(Libevent CONFIG REQUIRED) ++list(APPEND FOLLY_LINK_LIBRARIES libevent::core libevent::extra) ++if(NOT WIN32) ++ list(APPEND FOLLY_LINK_LIBRARIES libevent::pthreads) ++endif() ++ +if (CMAKE_REQUIRE_FIND_PACKAGE_ZLIB) -+ find_package(ZLIB MODULE REQUIRED) -+ set(FOLLY_HAVE_LIBZ ${ZLIB_FOUND}) -+ if (ZLIB_FOUND) -+ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) -+ list(APPEND FOLLY_LINK_LIBRARIES ${ZLIB_LIBRARIES}) -+ list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES}) -+ endif() ++ find_package(ZLIB MODULE REQUIRED) ++ set(FOLLY_HAVE_LIBZ ${ZLIB_FOUND}) ++ if (ZLIB_FOUND) ++ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) ++ list(APPEND FOLLY_LINK_LIBRARIES ${ZLIB_LIBRARIES}) ++ list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES}) ++ endif() endif() find_package(OpenSSL 1.1.1 MODULE REQUIRED) -@@ -104,25 +105,30 @@ if (LIBLZMA_FOUND) +@@ -103,25 +103,30 @@ if (LIBLZMA_FOUND) list(APPEND FOLLY_LINK_LIBRARIES ${LIBLZMA_LIBRARIES}) endif() @@ -125,7 +122,7 @@ index 4b78e9f02..eb77e29c9 100644 -if (LZ4_FOUND) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LZ4_INCLUDE_DIR}) - list(APPEND FOLLY_LINK_LIBRARIES ${LZ4_LIBRARY}) -+if (NOT CMAKE_DISABLE_FIND_PACKAGE_LZ4) ++if(NOT CMAKE_DISABLE_FIND_PACKAGE_LZ4) + find_package(lz4 CONFIG) + if(TARGET lz4::lz4) + set(FOLLY_HAVE_LIBLZ4 1) @@ -138,15 +135,15 @@ index 4b78e9f02..eb77e29c9 100644 -if(ZSTD_FOUND) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIR}) - list(APPEND FOLLY_LINK_LIBRARIES ${ZSTD_LIBRARY}) -+if (NOT CMAKE_DISABLE_FIND_PACKAGE_Zstd) -+ find_package(zstd CONFIG) -+ if(TARGET zstd::libzstd_shared ) -+ set(FOLLY_HAVE_LIBZSTD 1) -+ list(APPEND FOLLY_LINK_LIBRARIES zstd::libzstd_shared) -+ elseif( TARGET zstd::libzstd_static ) -+ set(FOLLY_HAVE_LIBZSTD 1) -+ list(APPEND FOLLY_LINK_LIBRARIES zstd::libzstd_static) -+ endif() ++if(NOT CMAKE_DISABLE_FIND_PACKAGE_Zstd) ++ find_package(zstd CONFIG) ++ if(TARGET zstd::libzstd_shared ) ++ set(FOLLY_HAVE_LIBZSTD 1) ++ list(APPEND FOLLY_LINK_LIBRARIES zstd::libzstd_shared) ++ elseif( TARGET zstd::libzstd_static ) ++ set(FOLLY_HAVE_LIBZSTD 1) ++ list(APPEND FOLLY_LINK_LIBRARIES zstd::libzstd_static) ++ endif() endif() -find_package(Snappy MODULE) @@ -154,49 +151,62 @@ index 4b78e9f02..eb77e29c9 100644 -if (SNAPPY_FOUND) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${SNAPPY_INCLUDE_DIR}) - list(APPEND FOLLY_LINK_LIBRARIES ${SNAPPY_LIBRARY}) -+if (NOT CMAKE_DISABLE_FIND_PACKAGE_Snappy) -+ find_package(Snappy CONFIG) -+ if(TARGET Snappy::snappy) -+ list(APPEND FOLLY_LINK_LIBRARIES Snappy::snappy) -+ endif() ++if(NOT CMAKE_DISABLE_FIND_PACKAGE_Snappy) ++ find_package(Snappy CONFIG) ++ if(TARGET Snappy::snappy) ++ list(APPEND FOLLY_LINK_LIBRARIES Snappy::snappy) ++ endif() endif() find_package(LibDwarf) -@@ -137,13 +143,18 @@ find_package(LibAIO) - list(APPEND FOLLY_LINK_LIBRARIES ${LIBAIO_LIBRARIES}) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBAIO_INCLUDE_DIRS}) +@@ -132,17 +137,24 @@ find_package(Libiberty) + list(APPEND FOLLY_LINK_LIBRARIES ${LIBIBERTY_LIBRARIES}) + list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBIBERTY_INCLUDE_DIRS}) + +-find_package(LibAIO) +-list(APPEND FOLLY_LINK_LIBRARIES ${LIBAIO_LIBRARIES}) +-list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBAIO_INCLUDE_DIRS}) ++if(WITH_libaio) ++ find_package(LibAIO) ++ list(APPEND FOLLY_LINK_LIBRARIES ${LIBAIO_LIBRARIES}) ++ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBAIO_INCLUDE_DIRS}) ++endif() +-find_package(LibUring) +-list(APPEND FOLLY_LINK_LIBRARIES ${LIBURING_LIBRARIES}) +-list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBURING_INCLUDE_DIRS}) +if(WITH_liburing) - find_package(LibUring) - list(APPEND FOLLY_LINK_LIBRARIES ${LIBURING_LIBRARIES}) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBURING_INCLUDE_DIRS}) ++ find_package(LibUring) ++ list(APPEND FOLLY_LINK_LIBRARIES ${LIBURING_LIBRARIES}) ++ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBURING_INCLUDE_DIRS}) +endif() -find_package(Libsodium) -list(APPEND FOLLY_LINK_LIBRARIES ${LIBSODIUM_LIBRARIES}) -list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBSODIUM_INCLUDE_DIRS}) -+if (NOT CMAKE_DISABLE_FIND_PACKAGE_unofficial-sodium) -+ find_package(unofficial-sodium CONFIG) -+ if(TARGET unofficial-sodium::sodium) -+ list(APPEND FOLLY_LINK_LIBRARIES unofficial-sodium::sodium) -+ endif() ++if(NOT CMAKE_DISABLE_FIND_PACKAGE_unofficial-sodium) ++ find_package(unofficial-sodium CONFIG) ++ if(TARGET unofficial-sodium::sodium) ++ list(APPEND FOLLY_LINK_LIBRARIES unofficial-sodium::sodium) ++ endif() +endif() list(APPEND FOLLY_LINK_LIBRARIES ${CMAKE_DL_LIBS}) list(APPEND CMAKE_REQUIRED_LIBRARIES ${CMAKE_DL_LIBS}) -@@ -154,9 +165,9 @@ if (PYTHON_EXTENSIONS) +@@ -153,10 +165,10 @@ if (PYTHON_EXTENSIONS) endif () find_package(LibUnwind) -list(APPEND FOLLY_LINK_LIBRARIES ${LIBUNWIND_LIBRARIES}) -list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBUNWIND_INCLUDE_DIRS}) if (LIBUNWIND_FOUND) + set(FOLLY_HAVE_LIBUNWIND ON) + list(APPEND FOLLY_LINK_LIBRARIES ${LIBUNWIND_LIBRARIES}) + list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBUNWIND_INCLUDE_DIRS}) - set(FOLLY_HAVE_LIBUNWIND ON) endif() if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") -@@ -299,11 +310,7 @@ endif() + list(APPEND FOLLY_LINK_LIBRARIES "execinfo") +@@ -298,11 +310,7 @@ endif() add_library(folly_deps INTERFACE) diff --git a/dev/vcpkg/ports/folly/fix-unistd-include.patch b/dev/vcpkg/ports/folly/fix-unistd-include.patch new file mode 100644 index 000000000000..9e3fe5d7a720 --- /dev/null +++ b/dev/vcpkg/ports/folly/fix-unistd-include.patch @@ -0,0 +1,13 @@ +diff --git a/folly/experimental/symbolizer/Elf.cpp b/folly/experimental/symbolizer/Elf.cpp +index 6a7f74f..6546fe0 100644 +--- a/folly/experimental/symbolizer/Elf.cpp ++++ b/folly/experimental/symbolizer/Elf.cpp +@@ -49,6 +49,8 @@ + #define FOLLY_ELF_NATIVE_CLASS __WORDSIZE + #endif // __ELF_NATIVE_CLASS + ++#include ++ + namespace folly { + namespace symbolizer { + diff --git a/dev/vcpkg/ports/folly/portfile.cmake b/dev/vcpkg/ports/folly/portfile.cmake index 12d503331821..40a0231ea31f 100644 --- a/dev/vcpkg/ports/folly/portfile.cmake +++ b/dev/vcpkg/ports/folly/portfile.cmake @@ -8,15 +8,15 @@ vcpkg_add_to_path("${PYTHON3_DIR}") vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO facebook/folly - REF "v${VERSION}" #v2023.12.04.00 - SHA512 5fe3d99dbd4f10698cc3f5076f80f627932c5f527210fac5610f60e373b90ce0ce44655e4be8467dbafcd007d6b82ec05f4b4014ff4697c5e144d6a2105a3c92 + REF "v${VERSION}" + SHA512 6f4fa9b30ff0aba4c1c89619ada2cafd8943ba9a06b6ac4086fba212f26f0df606c8735485110eec6977900d45fc33ddcfdb8095dc6728b8c12b5cac65672e12 HEAD_REF main PATCHES - reorder-glog-gflags.patch - disable-non-underscore-posix-names.patch boost-1.70.patch fix-windows-minmax.patch fix-deps.patch + disable-uninitialized-resize-on-new-stl.patch + fix-unistd-include.patch ) file(REMOVE "${SOURCE_PATH}/CMake/FindFmt.cmake") @@ -29,6 +29,7 @@ file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindGMock.cmake") file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindGflags.cmake") file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindGlog.cmake") file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindLibEvent.cmake") +file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindLibUnwind.cmake") file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindSodium.cmake") file(REMOVE "${SOURCE_PATH}/build/fbcode_builder/CMake/FindZstd.cmake") @@ -42,6 +43,7 @@ vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS FEATURES "zlib" CMAKE_REQUIRE_FIND_PACKAGE_ZLIB "liburing" WITH_liburing + "libaio" WITH_libaio INVERTED_FEATURES "bzip2" CMAKE_DISABLE_FIND_PACKAGE_BZip2 "lzma" CMAKE_DISABLE_FIND_PACKAGE_LibLZMA @@ -54,19 +56,19 @@ vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS vcpkg_cmake_configure( SOURCE_PATH "${SOURCE_PATH}" OPTIONS + # Exclude exception tracer, which is necessary to statically link libstdc++. + # https://github.com/facebook/folly/issues/1623 + -DFOLLY_NO_EXCEPTION_TRACER=ON -DMSVC_USE_STATIC_RUNTIME=${MSVC_USE_STATIC_RUNTIME} - # -DCMAKE_DISABLE_FIND_PACKAGE_LibDwarf=ON + -DCMAKE_DISABLE_FIND_PACKAGE_LibDwarf=OFF -DCMAKE_DISABLE_FIND_PACKAGE_Libiberty=ON -DCMAKE_DISABLE_FIND_PACKAGE_LibAIO=ON -DLIBAIO_FOUND=OFF + # Required by Velox. -DFOLLY_HAVE_INT128_T=ON -DCMAKE_INSTALL_DIR=share/folly - # Fix folly static link libstdc++ - # See: https://github.com/facebook/folly/blob/b88123c2abf4b3244ed285e6db0d4bea2d24f95f/CMakeLists.txt#L192 - -DFOLLY_NO_EXCEPTION_TRACER=ON ${FEATURE_OPTIONS} MAYBE_UNUSED_VARIABLES - LIBAIO_FOUND MSVC_USE_STATIC_RUNTIME ) diff --git a/dev/vcpkg/ports/folly/reorder-glog-gflags.patch b/dev/vcpkg/ports/folly/reorder-glog-gflags.patch deleted file mode 100644 index 911481d8854c..000000000000 --- a/dev/vcpkg/ports/folly/reorder-glog-gflags.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/CMake/folly-deps.cmake b/CMake/folly-deps.cmake -index 92b8ce7..04589ed 100644 ---- a/CMake/folly-deps.cmake -+++ b/CMake/folly-deps.cmake -@@ -52,6 +52,11 @@ find_package(DoubleConversion MODULE REQUIRED) - list(APPEND FOLLY_LINK_LIBRARIES ${DOUBLE_CONVERSION_LIBRARY}) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${DOUBLE_CONVERSION_INCLUDE_DIR}) - -+find_package(Glog CONFIG REQUIRED) -+set(FOLLY_HAVE_LIBGLOG ${GLOG_FOUND}) -+list(APPEND FOLLY_LINK_LIBRARIES glog::glog) -+list(APPEND FOLLY_INCLUDE_DIRECTORIES ${GLOG_INCLUDE_DIR}) -+ - find_package(Gflags MODULE) - set(FOLLY_HAVE_LIBGFLAGS ${LIBGFLAGS_FOUND}) - if(LIBGFLAGS_FOUND) -@@ -61,11 +66,6 @@ if(LIBGFLAGS_FOUND) - set(FOLLY_LIBGFLAGS_INCLUDE ${LIBGFLAGS_INCLUDE_DIR}) - endif() - --find_package(Glog MODULE) --set(FOLLY_HAVE_LIBGLOG ${GLOG_FOUND}) --list(APPEND FOLLY_LINK_LIBRARIES ${GLOG_LIBRARY}) --list(APPEND FOLLY_INCLUDE_DIRECTORIES ${GLOG_INCLUDE_DIR}) -- - find_package(LibEvent MODULE REQUIRED) - list(APPEND FOLLY_LINK_LIBRARIES ${LIBEVENT_LIB}) - list(APPEND FOLLY_INCLUDE_DIRECTORIES ${LIBEVENT_INCLUDE_DIR}) diff --git a/dev/vcpkg/ports/folly/vcpkg-cmake-wrapper.cmake b/dev/vcpkg/ports/folly/vcpkg-cmake-wrapper.cmake index f5997e836468..cce45a2e43e6 100644 --- a/dev/vcpkg/ports/folly/vcpkg-cmake-wrapper.cmake +++ b/dev/vcpkg/ports/folly/vcpkg-cmake-wrapper.cmake @@ -1,6 +1,6 @@ -set(FOLLY_PREV_MODULE_PATH ${CMAKE_MODULE_PATH}) -list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}) - -_find_package(${ARGS}) - +set(FOLLY_PREV_MODULE_PATH ${CMAKE_MODULE_PATH}) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}) + +_find_package(${ARGS}) + set(CMAKE_MODULE_PATH ${FOLLY_PREV_MODULE_PATH}) \ No newline at end of file diff --git a/dev/vcpkg/ports/folly/vcpkg.json b/dev/vcpkg/ports/folly/vcpkg.json index cbb1006baf57..b908f40dd9ea 100644 --- a/dev/vcpkg/ports/folly/vcpkg.json +++ b/dev/vcpkg/ports/folly/vcpkg.json @@ -1,7 +1,6 @@ { "name": "folly", - "version-string": "2023.12.04.00", - "port-version": 4, + "version-string": "2024.04.01.00", "description": "An open-source C++ library developed and used at Facebook. The library is UNSTABLE on Windows", "homepage": "https://github.com/facebook/folly", "license": "Apache-2.0", @@ -19,14 +18,13 @@ "boost-smart-ptr", "boost-system", "boost-thread", + "boost-variant", "double-conversion", "fmt", "gflags", "glog", "libevent", "openssl", - "libdwarf", - "libelf", { "name": "vcpkg-cmake", "host": true @@ -46,6 +44,10 @@ "bzip2" ] }, + "libaio": { + "description": "Support compile with libaio", + "supports": "linux & x64" + }, "libsodium": { "description": "Support libsodium for cryto", "dependencies": [ From d7374bd8f5a2ccc518d095ebd675e2ca3269ec47 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 28 May 2024 20:00:21 +0800 Subject: [PATCH 164/402] [GLUTEN-5314][VL] Separate FileSink instantiation for different file systems (#5881) --- cpp/velox/operators/writer/VeloxParquetDatasource.cc | 5 ++++- cpp/velox/operators/writer/VeloxParquetDatasource.h | 1 + cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h | 4 ++-- cpp/velox/operators/writer/VeloxParquetDatasourceGCS.h | 4 ++-- cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h | 4 ++-- cpp/velox/operators/writer/VeloxParquetDatasourceS3.h | 4 ++-- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cpp/velox/operators/writer/VeloxParquetDatasource.cc b/cpp/velox/operators/writer/VeloxParquetDatasource.cc index 2677b0a812d1..16558229e765 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasource.cc +++ b/cpp/velox/operators/writer/VeloxParquetDatasource.cc @@ -43,13 +43,16 @@ namespace { const int32_t kGzipWindowBits4k = 12; } -void VeloxParquetDatasource::init(const std::unordered_map& sparkConfs) { +void VeloxParquetDatasource::initSink(const std::unordered_map& /* sparkConfs */) { if (strncmp(filePath_.c_str(), "file:", 5) == 0) { sink_ = dwio::common::FileSink::create(filePath_, {.pool = pool_.get()}); } else { throw std::runtime_error("The file path is not local when writing data with parquet format in velox runtime!"); } +} +void VeloxParquetDatasource::init(const std::unordered_map& sparkConfs) { + initSink(sparkConfs); ArrowSchema cSchema{}; arrow::Status status = arrow::ExportSchema(*(schema_.get()), &cSchema); if (!status.ok()) { diff --git a/cpp/velox/operators/writer/VeloxParquetDatasource.h b/cpp/velox/operators/writer/VeloxParquetDatasource.h index 3df444016beb..12cf2c301a39 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasource.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasource.h @@ -89,6 +89,7 @@ class VeloxParquetDatasource : public Datasource { : Datasource(filePath, schema), filePath_(filePath), schema_(schema), pool_(std::move(veloxPool)) {} void init(const std::unordered_map& sparkConfs) override; + virtual void initSink(const std::unordered_map& sparkConfs); void inspectSchema(struct ArrowSchema* out) override; void write(const std::shared_ptr& cb) override; void close() override; diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h index 208e6a7ec55c..82e8f794cbce 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h @@ -41,13 +41,13 @@ class VeloxParquetDatasourceABFS final : public VeloxParquetDatasource { std::shared_ptr sinkPool, std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} - void init(const std::unordered_map& sparkConfs) override { + + void initSink(const std::unordered_map& sparkConfs) override { auto hiveConf = getHiveConfig(std::make_shared(sparkConfs)); auto fileSystem = filesystems::getFileSystem(filePath_, hiveConf); auto* abfsFileSystem = dynamic_cast(fileSystem.get()); sink_ = std::make_unique( abfsFileSystem->openFileForWrite(filePath_, {{}, sinkPool_.get()}), filePath_); - VeloxParquetDatasource::init(sparkConfs); } }; } // namespace gluten diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceGCS.h b/cpp/velox/operators/writer/VeloxParquetDatasourceGCS.h index b8a9b5431df6..0c2bfa213892 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceGCS.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceGCS.h @@ -40,12 +40,12 @@ class VeloxParquetDatasourceGCS final : public VeloxParquetDatasource { std::shared_ptr sinkPool, std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} - void init(const std::unordered_map& sparkConfs) override { + + void initSink(const std::unordered_map& /* sparkConfs */) override { auto fileSystem = filesystems::getFileSystem(filePath_, nullptr); auto* gcsFileSystem = dynamic_cast(fileSystem.get()); sink_ = std::make_unique( gcsFileSystem->openFileForWrite(filePath_, {{}, sinkPool_.get()}), filePath_); - VeloxParquetDatasource::init(sparkConfs); } }; } // namespace gluten diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h b/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h index 32cf960cbf2f..7722c8e51993 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h @@ -41,10 +41,10 @@ class VeloxParquetDatasourceHDFS final : public VeloxParquetDatasource { std::shared_ptr sinkPool, std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} - void init(const std::unordered_map& sparkConfs) override { + + void initSink(const std::unordered_map& sparkConfs) override { auto hiveConf = getHiveConfig(std::make_shared(sparkConfs)); sink_ = dwio::common::FileSink::create(filePath_, {.connectorProperties = hiveConf, .pool = sinkPool_.get()}); - VeloxParquetDatasource::init(sparkConfs); } }; } // namespace gluten diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h b/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h index a5c49fcd9f81..3231a8a1ee5c 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h +++ b/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h @@ -41,10 +41,10 @@ class VeloxParquetDatasourceS3 final : public VeloxParquetDatasource { std::shared_ptr sinkPool, std::shared_ptr schema) : VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {} - void init(const std::unordered_map& sparkConfs) override { + + void initSink(const std::unordered_map& sparkConfs) override { auto hiveConf = getHiveConfig(std::make_shared(sparkConfs)); sink_ = dwio::common::FileSink::create(filePath_, {.connectorProperties = hiveConf, .pool = sinkPool_.get()}); - VeloxParquetDatasource::init(sparkConfs); } }; } // namespace gluten From 79c681b2847fccd43292d3d735bc3972eacb3b52 Mon Sep 17 00:00:00 2001 From: WangGuangxin Date: Wed, 29 May 2024 10:26:32 +0800 Subject: [PATCH 165/402] [GLUTEN-4422][CORE] Fix core dump caused by spill on closed iterator (#5874) Closes #4422 --- cpp/core/jni/JniWrapper.cc | 4 ++++ .../apache/gluten/vectorized/ColumnarBatchOutIterator.java | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index a04ba73a18af..f5a6c4bd70d0 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -544,6 +544,10 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterat auto ctx = gluten::getRuntime(env, wrapper); auto it = ctx->objectStore()->retrieve(iterHandle); + if (it == nullptr) { + std::string errorMessage = "Invalid result iter handle " + std::to_string(iterHandle); + throw gluten::GlutenException(errorMessage); + } return it->spillFixedSize(size); JNI_METHOD_END(kInvalidResourceHandle) } diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index 82b398439cde..37de9894392c 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -80,7 +80,11 @@ public IMetrics getMetricsInternal() throws IOException, ClassNotFoundException } public long spill(long size) { - return nativeSpill(iterHandle, size); + if (!closed.get()) { + return nativeSpill(iterHandle, size); + } else { + return 0L; + } } @Override From 2a81010d864ba9d716cc3d974c2ccac8354df5c6 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Wed, 29 May 2024 11:13:08 +0800 Subject: [PATCH 166/402] [VL] Enable partial merge mode for HLL (#5754) [VL] Enabl partial merge mode for HLL. --- .../HashAggregateExecTransformer.scala | 37 +++---------------- .../gluten/extension/HLLRewriteRule.scala | 22 ++--------- .../VeloxAggregateFunctionsSuite.scala | 34 ++--------------- 3 files changed, 14 insertions(+), 79 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala index 01ab56881936..4f33ae7c718c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala @@ -20,7 +20,6 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression._ import org.apache.gluten.expression.ConverterUtils.FunctionConfig -import org.apache.gluten.expression.aggregate.HLLAdapter import org.apache.gluten.substrait.`type`.{TypeBuilder, TypeNode} import org.apache.gluten.substrait.{AggregationParams, SubstraitContext} import org.apache.gluten.substrait.expression.{AggregateFunctionNode, ExpressionBuilder, ExpressionNode, ScalarFunctionNode} @@ -74,20 +73,6 @@ abstract class HashAggregateExecTransformer( TransformContext(childCtx.outputAttributes, output, relNode) } - override protected def checkAggFuncModeSupport( - aggFunc: AggregateFunction, - mode: AggregateMode): Boolean = { - aggFunc match { - case _: HLLAdapter => - mode match { - case Partial | Final => true - case _ => false - } - case _ => - super.checkAggFuncModeSupport(aggFunc, mode) - } - } - // Return whether the outputs partial aggregation should be combined for Velox computing. // When the partial outputs are multiple-column, row construct is needed. private def rowConstructNeeded(aggregateExpressions: Seq[AggregateExpression]): Boolean = { @@ -241,21 +226,21 @@ abstract class HashAggregateExecTransformer( } aggregateFunction match { - case hllAdapter: HLLAdapter => + case _ if aggregateFunction.aggBufferAttributes.size > 1 => + generateMergeCompanionNode() + case _ => aggregateMode match { - case Partial => - // For Partial mode output type is binary. + case Partial | PartialMerge => val partialNode = ExpressionBuilder.makeAggregateFunction( VeloxAggregateFunctionsBuilder.create(args, aggregateFunction, aggregateMode), childrenNodeList, modeKeyWord, ConverterUtils.getTypeNode( - hllAdapter.inputAggBufferAttributes.head.dataType, - hllAdapter.inputAggBufferAttributes.head.nullable) + aggregateFunction.inputAggBufferAttributes.head.dataType, + aggregateFunction.inputAggBufferAttributes.head.nullable) ) aggregateNodeList.add(partialNode) case Final => - // For Final mode output type is long. val aggFunctionNode = ExpressionBuilder.makeAggregateFunction( VeloxAggregateFunctionsBuilder.create(args, aggregateFunction, aggregateMode), childrenNodeList, @@ -266,16 +251,6 @@ abstract class HashAggregateExecTransformer( case other => throw new GlutenNotSupportException(s"$other is not supported.") } - case _ if aggregateFunction.aggBufferAttributes.size > 1 => - generateMergeCompanionNode() - case _ => - val aggFunctionNode = ExpressionBuilder.makeAggregateFunction( - VeloxAggregateFunctionsBuilder.create(args, aggregateFunction, aggregateMode), - childrenNodeList, - modeKeyWord, - ConverterUtils.getTypeNode(aggregateFunction.dataType, aggregateFunction.nullable) - ) - aggregateNodeList.add(aggFunctionNode) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala index 03819fc102ab..7bae64ff8d59 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala @@ -35,7 +35,7 @@ case class HLLRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { case hllExpr @ AggregateExpression(hll: HyperLogLogPlusPlus, _, _, _, _) if GlutenConfig.getConf.enableNativeHyperLogLogAggregateFunction && GlutenConfig.getConf.enableColumnarHashAgg && - !hasDistinctAggregateFunc(a) && isDataTypeSupported(hll.child.dataType) => + isDataTypeSupported(hll.child.dataType) => AggregateExpression( HLLAdapter( hll.child, @@ -51,29 +51,15 @@ case class HLLRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { } } - private def hasDistinctAggregateFunc(agg: Aggregate): Boolean = { - agg.aggregateExpressions - .flatMap(_.collect { case ae: AggregateExpression => ae }) - .exists(_.isDistinct) - } - private def isDataTypeSupported(dataType: DataType): Boolean = { // HLL in velox only supports below data types. we should not offload HLL to velox, if // child's data type is not supported. This prevents the case only partail agg is fallbacked. // As spark and velox have different HLL binary formats, HLL binary generated by spark can't // be parsed by velox, it would cause the error: 'Unexpected type of HLL'. dataType match { - case BooleanType => true - case ByteType => true - case _: CharType => true - case DateType => true - case DoubleType => true - case FloatType => true - case IntegerType => true - case LongType => true - case ShortType => true - case StringType => true - case _: DecimalType => true + case BooleanType | ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType | + StringType | _: CharType | _: DecimalType | DateType => + true case _ => false } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index ffed6373123e..4f6f4eb224d0 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -553,41 +553,15 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu } test("approx_count_distinct") { - runQueryAndCompare(""" - |select approx_count_distinct(l_shipmode) from lineitem; - |""".stripMargin) { - checkGlutenOperatorMatch[HashAggregateExecTransformer] - } runQueryAndCompare( - "select approx_count_distinct(l_partkey), count(distinct l_orderkey) from lineitem") { - df => - { - assert( - getExecutedPlan(df).count( - plan => { - plan.isInstanceOf[HashAggregateExecTransformer] - }) == 0) - } - } - } - - test("approx_count_distinct decimal") { - // The data type of l_discount is decimal. - runQueryAndCompare(""" - |select approx_count_distinct(l_discount) from lineitem; - |""".stripMargin) { + """ + |select approx_count_distinct(l_shipmode), approx_count_distinct(l_discount) from lineitem; + |""".stripMargin) { checkGlutenOperatorMatch[HashAggregateExecTransformer] } runQueryAndCompare( "select approx_count_distinct(l_discount), count(distinct l_orderkey) from lineitem") { - df => - { - assert( - getExecutedPlan(df).count( - plan => { - plan.isInstanceOf[HashAggregateExecTransformer] - }) == 0) - } + checkGlutenOperatorMatch[HashAggregateExecTransformer] } } From 4f6d690e6513804691a6ef12593420f6c7946445 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 29 May 2024 13:12:17 +0800 Subject: [PATCH 167/402] [VL] Keep gluten fat jar built out previously for other Spark versions (#5905) this patch can fixes the build packages behavior for all Spark version in a single run --- package/pom.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package/pom.xml b/package/pom.xml index f405124a2a77..db4056a7e109 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -205,8 +205,10 @@ target - *3.2* - *3.3* + *spark3.2* + *spark3.3* + *spark3.4* + *spark3.5* false From efb81e2847da13cf51f5df11cea41b083b8f0475 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 29 May 2024 16:39:35 +0800 Subject: [PATCH 168/402] [VL] Gluten-it: Improve test report table rendering (#5889) --- .../integration/command/SparkRunModes.java | 3 + .../integration/action/Parameterized.scala | 28 +- .../gluten/integration/action/Queries.scala | 23 +- .../integration/action/QueriesCompare.scala | 50 +-- .../integration/action/TableFormatter.scala | 78 ----- .../integration/action/TableRender.scala | 308 ++++++++++++++++++ .../integration/action/TableRenderTest.scala | 91 ++++++ 7 files changed, 454 insertions(+), 127 deletions(-) delete mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala create mode 100644 tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java index f5a5c73a682f..cfd3848d8158 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/SparkRunModes.java @@ -140,6 +140,9 @@ public Map extraSparkConf() { Optional extraClassPath = Arrays.stream(classPathValues).filter(classPath -> { File file = new File(classPath); return file.exists() && file.isFile() && extraJarSet.contains(file.getName()); + }).map(classPath -> { + File file = new File(classPath); + return file.getAbsolutePath(); }).reduce((s1, s2) -> s1 + File.pathSeparator + s2); final Map extras = new HashMap<>(); diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala index 2871ef2de0f8..799b7632e02c 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala @@ -18,13 +18,14 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.gluten.integration.action.Actions.QuerySelector +import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} import org.apache.spark.sql.ConfUtils.ConfImplicits._ import org.apache.spark.sql.SparkSessionSwitcher import scala.collection.mutable -import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import scala.collection.mutable.ListBuffer class Parameterized( scale: Double, @@ -198,24 +199,25 @@ case class TestResultLine( object TestResultLine { class Parser(dimNames: Seq[String], metricNames: Seq[String]) - extends TableFormatter.RowParser[TestResultLine] { - override def parse(line: TestResultLine): Seq[Any] = { - val values = ArrayBuffer[Any](line.queryId, line.succeed) + extends TableRender.RowParser[TestResultLine] { + override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { + val inc = rowAppender.incremental() + inc.next().write(line.queryId) + inc.next().write(line.succeed) dimNames.foreach { dimName => val coordinate = line.coordinate.coordinate if (!coordinate.contains(dimName)) { throw new IllegalStateException("Dimension name not found" + dimName) } - values.append(coordinate(dimName)) + inc.next().write(coordinate(dimName)) } metricNames.foreach { metricName => val metrics = line.metrics - values.append(metrics.getOrElse(metricName, "N/A")) + inc.next().write(metrics.getOrElse(metricName, "N/A")) } - values.append(line.rowCount.getOrElse("N/A")) - values.append(line.planningTimeMillis.getOrElse("N/A")) - values.append(line.executionTimeMillis.getOrElse("N/A")) - values + inc.next().write(line.rowCount.getOrElse("N/A")) + inc.next().write(line.planningTimeMillis.getOrElse("N/A")) + inc.next().write(line.executionTimeMillis.getOrElse("N/A")) } } } @@ -231,14 +233,14 @@ case class TestResultLines( fields.append("Row Count") fields.append("Planning Time (Millis)") fields.append("Query Time (Millis)") - val formatter = TableFormatter.create[TestResultLine](fields: _*)( + val render = TableRender.plain[TestResultLine](fields: _*)( new TestResultLine.Parser(dimNames, metricNames)) lines.foreach { line => - formatter.appendRow(line) + render.appendRow(line) } - formatter.print(System.out) + render.print(System.out) } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala index cf24b906b2aa..540abbf454c3 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala @@ -18,6 +18,7 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.gluten.integration.action.Actions.QuerySelector +import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat import org.apache.gluten.integration.{QueryRunner, Suite} @@ -108,20 +109,20 @@ object Queries { errorMessage: Option[String]) object TestResultLine { - implicit object Parser extends TableFormatter.RowParser[TestResultLine] { - override def parse(line: TestResultLine): Seq[Any] = { - Seq( - line.queryId, - line.testPassed, - line.rowCount.getOrElse("N/A"), - line.planningTimeMillis.getOrElse("N/A"), - line.executionTimeMillis.getOrElse("N/A")) + implicit object Parser extends TableRender.RowParser[TestResultLine] { + override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { + val inc = rowAppender.incremental() + inc.next().write(line.queryId) + inc.next().write(line.testPassed) + inc.next().write(line.rowCount.getOrElse("N/A")) + inc.next().write(line.planningTimeMillis.getOrElse("N/A")) + inc.next().write(line.executionTimeMillis.getOrElse("N/A")) } } } private def printResults(results: List[TestResultLine]): Unit = { - val formatter = TableFormatter.create[TestResultLine]( + val render = TableRender.plain[TestResultLine]( "Query ID", "Was Passed", "Row Count", @@ -129,10 +130,10 @@ object Queries { "Query Time (Millis)") results.foreach { line => - formatter.appendRow(line) + render.appendRow(line) } - formatter.print(System.out) + render.print(System.out) } private def aggregate(succeed: List[TestResultLine], name: String): List[TestResultLine] = { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala index 320bd61b609d..596c293e473e 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala @@ -18,6 +18,7 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.gluten.integration.action.Actions.QuerySelector +import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} import org.apache.spark.sql.{SparkSessionSwitcher, TestUtils} @@ -111,45 +112,44 @@ object QueriesCompare { errorMessage: Option[String]) object TestResultLine { - implicit object Parser extends TableFormatter.RowParser[TestResultLine] { - override def parse(line: TestResultLine): Seq[Any] = { - val timeVariation = + implicit object Parser extends TableRender.RowParser[TestResultLine] { + override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { + val inc = rowAppender.incremental() + val speedUp = if (line.expectedExecutionTimeMillis.nonEmpty && line.actualExecutionTimeMillis.nonEmpty) { Some( ((line.expectedExecutionTimeMillis.get - line.actualExecutionTimeMillis.get).toDouble / line.actualExecutionTimeMillis.get.toDouble) * 100) } else None - Seq( - line.queryId, - line.testPassed, - line.expectedRowCount.getOrElse("N/A"), - line.actualRowCount.getOrElse("N/A"), - line.expectedPlanningTimeMillis.getOrElse("N/A"), - line.actualPlanningTimeMillis.getOrElse("N/A"), - line.expectedExecutionTimeMillis.getOrElse("N/A"), - line.actualExecutionTimeMillis.getOrElse("N/A"), - timeVariation.map("%15.2f%%".format(_)).getOrElse("N/A")) + inc.next().write(line.queryId) + inc.next().write(line.testPassed) + inc.next().write(line.expectedRowCount.getOrElse("N/A")) + inc.next().write(line.actualRowCount.getOrElse("N/A")) + inc.next().write(line.expectedPlanningTimeMillis.getOrElse("N/A")) + inc.next().write(line.actualPlanningTimeMillis.getOrElse("N/A")) + inc.next().write(line.expectedExecutionTimeMillis.getOrElse("N/A")) + inc.next().write(line.actualExecutionTimeMillis.getOrElse("N/A")) + inc.next().write(speedUp.map("%.2f%%".format(_)).getOrElse("N/A")) } } } private def printResults(results: List[TestResultLine]): Unit = { - val formatter = TableFormatter.create[TestResultLine]( - "Query ID", - "Was Passed", - "Expected Row Count", - "Actual Row Count", - "Baseline Planning Time (Millis)", - "Planning Time (Millis)", - "Baseline Query Time (Millis)", - "Query Time (Millis)", - "Query Time Variation") + import org.apache.gluten.integration.action.TableRender.Field._ + + val render = TableRender.create[TestResultLine]( + Leaf("Query ID"), + Leaf("Passed"), + Branch("Row Count", List(Leaf("Vanilla"), Leaf("Gluten"))), + Branch("Planning Time (Millis)", List(Leaf("Vanilla"), Leaf("Gluten"))), + Branch("Query Time (Millis)", List(Leaf("Vanilla"), Leaf("Gluten"))), + Leaf("Speedup")) results.foreach { line => - formatter.appendRow(line) + render.appendRow(line) } - formatter.print(System.out) + render.print(System.out) } private def aggregate(succeed: List[TestResultLine], name: String): List[TestResultLine] = { diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala deleted file mode 100644 index 07e253d5e4e1..000000000000 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableFormatter.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.gluten.integration.action - -import java.io.{OutputStream, PrintStream} -import scala.collection.mutable - -trait TableFormatter[ROW <: Any] { - import TableFormatter._ - def appendRow(row: ROW): Unit - def print(s: OutputStream): Unit -} - -object TableFormatter { - def create[ROW <: Any](fields: String*)( - implicit parser: RowParser[ROW]): TableFormatter[ROW] = { - assert(fields.nonEmpty) - new Impl[ROW](Schema(fields), parser) - } - - private case class Schema(fields: Seq[String]) - - private class Impl[ROW <: Any](schema: Schema, parser: RowParser[ROW]) - extends TableFormatter[ROW] { - private val rows = mutable.ListBuffer[Seq[String]]() - - override def appendRow(row: ROW): Unit = { - val parsed = parser.parse(row) - assert(parsed.size == schema.fields.size) - rows += parsed.map(_.toString) - } - - override def print(s: OutputStream): Unit = { - val printer = new PrintStream(s) - if (rows.isEmpty) { - printer.println("(N/A)") - printer.flush() - return - } - val numFields = schema.fields.size - val widths = (0 until numFields) - .map { i => - rows.map(_(i).length).max max schema.fields(i).length - } - .map(_ + 1) - val pBuilder = StringBuilder.newBuilder - pBuilder ++= "|" - widths.foreach { w => - pBuilder ++= s"%${w}s|" - } - val pattern = pBuilder.toString() - printer.println(String.format(pattern, schema.fields: _*)) - rows.foreach { r => - printer.println(String.format(pattern, r: _*)) - } - printer.flush() - } - } - - trait RowParser[ROW <: Any] { - def parse(row: ROW): Seq[Any] - } -} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala new file mode 100644 index 000000000000..b25a5db93278 --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gluten.integration.action + +import org.apache.commons.lang3.StringUtils +import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender + +import java.io.{OutputStream, PrintStream} +import scala.collection.mutable + +trait TableRender[ROW <: Any] { + def appendRow(row: ROW): Unit + def print(s: OutputStream): Unit +} + +object TableRender { + def create[ROW <: Any](fields: Field*)(implicit parser: RowParser[ROW]): TableRender[ROW] = { + assert(fields.nonEmpty) + new Impl[ROW](Schema(fields), parser) + } + + def plain[ROW <: Any](fields: String*)(implicit parser: RowParser[ROW]): TableRender[ROW] = { + assert(fields.nonEmpty) + new Impl[ROW](Schema(fields.map(Field.Leaf)), parser) + } + + trait Field { + def name: String + def leafs: Seq[Field.Leaf] + } + + object Field { + case class Branch(override val name: String, children: Seq[Field]) extends Field { + override val leafs: Seq[Leaf] = { + children.map(leafsOf).reduce(_ ++ _) + } + + private def leafsOf(field: Field): Seq[Field.Leaf] = { + field match { + case l @ Field.Leaf(_) => List(l) + case b @ Field.Branch(_, children) => + children.map(child => leafsOf(child)).reduce(_ ++ _) + } + } + } + case class Leaf(override val name: String) extends Field { + override val leafs: Seq[Leaf] = List(this) + } + } + + private case class Schema(fields: Seq[Field]) { + val leafs: Seq[Field.Leaf] = { + fields.map(_.leafs).reduce(_ ++ _) + } + + val maxNestingLevel: Int = { + fields.map(maxNestingLevelOf).max + } + + private def maxNestingLevelOf(field: Field): Int = { + field match { + case _: Field.Leaf => 1 + case Field.Branch(_, children) => children.map(maxNestingLevelOf).max + 1 + } + } + } + + private class Impl[ROW <: Any](schema: Schema, parser: RowParser[ROW]) + extends TableRender[ROW] { + private val appenderFactory = RowParser.FieldAppender.TableAppender.create(schema) + + override def appendRow(row: ROW): Unit = { + parser.parse(appenderFactory.newRow(), row) + } + + override def print(s: OutputStream): Unit = { + val data = appenderFactory.data() + val printer = new PrintStream(s) + if (data.isEmpty) { + printer.println("(N/A)") + printer.flush() + return + } + + // The map is incrementally updated while walking the schema tree from top down. + val widthMap: mutable.Map[Int, Int] = mutable.Map() + + val dataWidths = schema.leafs.indices + .map { i => + data.map(_(i).length).max + } + .map(_ + 2) + + schema.leafs.zipWithIndex.foreach { + case (leaf, i) => + val dataWidth = dataWidths(i) + widthMap += (System.identityHashCode(leaf) -> (dataWidth max (leaf.name.length + 2))) + } + + schema.fields.foreach { root => + def updateWidth(field: Field, lowerBound: Int): Unit = { + field match { + case branch @ Field.Branch(name, children) => + val childLowerBound = + Math.ceil((lowerBound max name.length + 2).toDouble / children.size.toDouble).toInt + children.foreach(child => updateWidth(child, childLowerBound)) + val childrenWidth = + children.map(child => widthMap(System.identityHashCode(child))).sum + val width = childLowerBound * children.size max childrenWidth + children.size - 1 + val hash = System.identityHashCode(branch) + widthMap += hash -> width + case leaf @ Field.Leaf(name) => + val hash = System.identityHashCode(leaf) + val newWidth = widthMap(hash) max lowerBound + widthMap.put(hash, newWidth) + case _ => new IllegalStateException() + } + } + + updateWidth(root, 0) + } + + trait SchemaCell + case class Given(field: Field) extends SchemaCell + case class PlaceHolder(leaf: Field.Leaf) extends SchemaCell + + (0 until schema.maxNestingLevel).foldRight[Seq[SchemaCell]](schema.fields.map(Given)) { + case (_, cells) => + val schemaLine = cells + .map { + case Given(field) => + (field.name, widthMap(System.identityHashCode(field))) + case PlaceHolder(leaf) => + ("", widthMap(System.identityHashCode(leaf))) + } + .map { + case (name, width) => + StringUtils.center(name, width) + } + .mkString("|", "|", "|") + printer.println(schemaLine) + cells.flatMap { f => + f match { + case Given(Field.Branch(name, children)) => children.map(Given) + case Given(l @ Field.Leaf(name)) => List(PlaceHolder(l)) + case p: PlaceHolder => List(p) + case _ => throw new IllegalStateException() + } + } + } + + val separationLine = schema.leafs + .map { leaf => + widthMap(System.identityHashCode(leaf)) + } + .map { width => + new String(Array.tabulate(width)(_ => '-')) + } + .mkString("|", "|", "|") + + printer.println(separationLine) + + data.foreach { row => + val dataLine = row + .zip(schema.leafs) + .map { + case (value, leaf) => + (value, widthMap(System.identityHashCode(leaf))) + } + .map { + case (value, width) => + StringUtils.leftPad(value, width) + } + .mkString("|", "|", "|") + printer.println(dataLine) + } + + printer.flush() + } + } + + trait RowParser[ROW <: Any] { + def parse(rowFactory: RowAppender, row: ROW): Unit + } + + object RowParser { + trait FieldAppender { + def child(name: String): FieldAppender + def write(value: Any): Unit + } + + object FieldAppender { + trait RowAppender { + def field(name: String): FieldAppender + def field(offset: Int): FieldAppender + def incremental(): RowAppender.Incremental + } + + object RowAppender { + def create( + schema: Schema, + mutableRows: mutable.ListBuffer[Array[String]]): RowAppender = { + new RowAppenderImpl(schema, mutableRows) + } + + trait Incremental { + def next(): FieldAppender + } + + private class RowAppenderImpl( + schema: Schema, + mutableRows: mutable.ListBuffer[Array[String]]) + extends RowAppender { + private val mutableRow = Array.tabulate(schema.leafs.size) { _ => + "UNFILLED" + } + mutableRows += mutableRow + + override def field(name: String): FieldAppender = { + val fields = schema.fields + assert(fields.count(_.name == name) == 1) + val field = fields.zipWithIndex.find(_._1.name == name).getOrElse { + throw new IllegalArgumentException(s"Field $name not found in $schema") + } + val column = field._2 + new FieldAppenderImpl(field._1, mutableRow, column) + } + + override def field(offset: Int): FieldAppender = { + new FieldAppenderImpl(schema.fields(offset), mutableRow, offset) + } + + override def incremental(): Incremental = { + new Incremental { + private var offset = 0 + override def next(): FieldAppender = { + val out = new FieldAppenderImpl(schema.leafs(offset), mutableRow, offset) + offset += 1 + out + } + } + } + } + } + + trait TableAppender { + def newRow(): RowAppender + def data(): Seq[Seq[String]] + } + + object TableAppender { + def create(schema: Schema): TableAppender = { + new TableAppenderImpl(schema) + } + + private class TableAppenderImpl(schema: Schema) extends TableAppender { + private val mutableRows: mutable.ListBuffer[Array[String]] = mutable.ListBuffer() + + override def newRow(): RowAppender = { + RowAppender.create(schema, mutableRows) + } + + override def data(): Seq[Seq[String]] = { + mutableRows.map(_.toSeq) + } + } + } + + private class FieldAppenderImpl(field: Field, mutableRow: Array[String], column: Int) + extends FieldAppender { + override def child(name: String): FieldAppender = { + field match { + case Field.Branch(_, children) => + assert(children.count(_.name == name) == 1) + val child = children.zipWithIndex.find(_._1.name == name).getOrElse { + throw new IllegalArgumentException(s"Field $name not found in $field") + } + val childField = child._1 + val childOffset = child._2 + new FieldAppenderImpl(childField, mutableRow, column + childOffset) + case _ => + throw new IllegalArgumentException(s"Field $field is not a branch") + } + } + + override def write(value: Any): Unit = { + assert(field.isInstanceOf[Field.Leaf]) + mutableRow(column) = value.toString + } + } + } + } +} diff --git a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala new file mode 100644 index 000000000000..87ad23f3622e --- /dev/null +++ b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gluten.integration.action + +import org.apache.gluten.integration.action.TableRender.Field._ +import org.apache.gluten.integration.action.TableRender.RowParser +import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender + +// The tests are manually run. +object TableRenderTest { + def case0(): Unit = { + val render: TableRender[Seq[String]] = TableRender.create( + Branch("ABC", List(Branch("AB", List(Leaf("A"), Leaf("B"))), Leaf("C"))), + Branch("DE", List(Leaf("D"), Leaf("E"))))(new RowParser[Seq[String]] { + override def parse(rowFactory: FieldAppender.RowAppender, row: Seq[String]): Unit = { + val inc = rowFactory.incremental() + row.foreach(ceil => inc.next().write(ceil)) + } + }) + render.print(Console.out) + Console.out.println() + } + + def case1(): Unit = { + val render: TableRender[Seq[String]] = TableRender.create( + Branch("ABC", List(Branch("AB", List(Leaf("A"), Leaf("B"))), Leaf("C"))), + Branch("DE", List(Leaf("D"), Leaf("E"))))(new RowParser[Seq[String]] { + override def parse(rowFactory: FieldAppender.RowAppender, row: Seq[String]): Unit = { + val inc = rowFactory.incremental() + row.foreach(ceil => inc.next().write(ceil)) + } + }) + + render.appendRow(List("aaaa", "b", "cccccc", "d", "eeeee")) + render.print(Console.out) + Console.out.println() + } + + def case2(): Unit = { + val render: TableRender[Seq[String]] = TableRender.create( + Branch("ABC", List(Branch("AAAAAAAAABBBBBB", List(Leaf("A"), Leaf("B"))), Leaf("C"))), + Branch("DE", List(Leaf("D"), Leaf("E"))))(new RowParser[Seq[String]] { + override def parse(rowFactory: FieldAppender.RowAppender, row: Seq[String]): Unit = { + val inc = rowFactory.incremental() + row.foreach(ceil => inc.next().write(ceil)) + } + }) + + render.appendRow(List("aaaa", "b", "cccccc", "d", "eeeee")) + render.print(Console.out) + Console.out.println() + } + + def case3(): Unit = { + val render: TableRender[Seq[String]] = TableRender.create( + Branch("ABC", List(Branch("AB", List(Leaf("A"), Leaf("B"))), Leaf("CCCCCCCCCCCCC"))), + Branch("DE", List(Leaf("D"), Leaf("E"))))(new RowParser[Seq[String]] { + override def parse(rowFactory: FieldAppender.RowAppender, row: Seq[String]): Unit = { + val inc = rowFactory.incremental() + row.foreach(ceil => inc.next().write(ceil)) + } + }) + + render.appendRow(List("aaaa", "b", "cccccc", "d", "eeeee")) + render.appendRow(List("aaaaaaaaaaaaa", "b", "cccccc", "ddddddddddd", "eeeee")) + render.print(Console.out) + Console.out.println() + } + + def main(args: Array[String]): Unit = { + case0() + case1() + case2() + case3() + } +} From 123c136728ed5529bca6a75488030687c788fbe9 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 29 May 2024 19:12:19 +0800 Subject: [PATCH 169/402] [VL] Daily Update Velox Version (2024_05_29) (#5903) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index b71a7ad47473..9fe130c1f69a 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_27 +VELOX_BRANCH=2024_05_29 VELOX_HOME="" #Set on run gluten on HDFS From f10d3b7bdd02dc28f85bf25207789251a265fa67 Mon Sep 17 00:00:00 2001 From: "shuai.xu" Date: Wed, 29 May 2024 19:35:52 +0800 Subject: [PATCH 170/402] [GLUTEN-5852][CH] fix mismatch result columns size exception (#5853) --- .../execution/GlutenClickhouseCountDistinctSuite.scala | 7 +++++++ .../org/apache/gluten/utils/PullOutProjectHelper.scala | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala index 1b954df22eac..5887050d0aaa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseCountDistinctSuite.scala @@ -213,4 +213,11 @@ class GlutenClickhouseCountDistinctSuite extends GlutenClickHouseWholeStageTrans } ) } + + test("GLUTEN-5852: Fix mismatch result columns size exception related to 5618") { + val sql = + "select distinct * from (select 2 as r3, count(distinct a, b, c), 2 as r1, 2 as r2 from " + + "values (0, null, 1), (1, 1, 1), (2, 2, 1), (1, 2, 1) ,(2, 2, 2) as data(a,b,c) group by c)" + compareResultsAgainstVanillaSpark(sql, true, { _ => }) + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala index 824694837dcb..505f13f263a2 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala @@ -61,7 +61,12 @@ trait PullOutProjectHelper { replaceBoundReference: Boolean = false): Expression = expr match { case alias: Alias => - projectExprsMap.getOrElseUpdate(alias.child.canonicalized, alias).toAttribute + alias.child match { + case _: Literal => + projectExprsMap.getOrElseUpdate(alias, alias).toAttribute + case _ => + projectExprsMap.getOrElseUpdate(alias.child.canonicalized, alias).toAttribute + } case attr: Attribute => attr case e: BoundReference if !replaceBoundReference => e case other => From 92798a2f85bf9d26e51e227929405958a91d9dc0 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 29 May 2024 20:35:59 +0800 Subject: [PATCH 171/402] [GLUTEN-4917][VL] CI: update GHA docker image (#5907) Update GHA docker image to add folly changes Signed-off-by: Yuan Zhou --- .github/workflows/velox_docker.yml | 2 +- .github/workflows/velox_docker_cache.yml | 2 +- dev/ci-velox-buildstatic.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 5c39fec35a78..d11d4032ffde 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -49,7 +49,7 @@ concurrency: jobs: build-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_05_22 # centos7 with dependencies installed + container: apache/gluten:gluten-vcpkg-builder_2024_05_29 # centos7 with dependencies installed steps: - uses: actions/checkout@v2 - name: Generate cache key diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 591be26093ee..44271c4fc0d0 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -27,7 +27,7 @@ concurrency: jobs: cache-native-lib: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_05_22 # centos7 with dependencies installed + container: apache/gluten:gluten-vcpkg-builder_2024_05_29 # centos7 with dependencies installed steps: - uses: actions/checkout@v2 - name: Generate cache key diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic.sh index 0754408169cf..208490d1c2eb 100755 --- a/dev/ci-velox-buildstatic.sh +++ b/dev/ci-velox-buildstatic.sh @@ -2,7 +2,7 @@ yum install sudo patch java-1.8.0-openjdk-devel -y cd $GITHUB_WORKSPACE/ep/build-velox/src ./get_velox.sh source /opt/rh/devtoolset-9/enable -source $GITHUB_WORKSPACE/dev/vcpkg/env.sh +source /opt/gluten/dev/vcpkg/env.sh cd $GITHUB_WORKSPACE/ sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt export NUM_THREADS=4 From 588faae351bff29d868336f530aa72eb99a57083 Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Wed, 29 May 2024 18:33:29 +0530 Subject: [PATCH 172/402] [VL] Enable SortShuffleSuite with ColumnarShuffleManager (#5816) --- .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 2 ++ .../apache/spark/GlutenSortShuffleSuite.scala | 24 +++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 2 ++ .../apache/spark/GlutenSortShuffleSuite.scala | 24 +++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 2 ++ .../apache/spark/GlutenSortShuffleSuite.scala | 24 +++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 2 ++ .../apache/spark/GlutenSortShuffleSuite.scala | 24 +++++++++++++++++++ 12 files changed, 108 insertions(+) create mode 100644 gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala create mode 100644 gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index afc427cd3664..2c34baa6379a 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -68,6 +68,7 @@ class ClickHouseTestSettings extends BackendTestSettings { false // nativeDoValidate failed due to spark conf cleanup case "GlutenDataSourceV2SQLSuite" => false // nativeDoValidate failed due to spark conf cleanup + case "GlutenSortShuffleSuite" => false case _ => true } preCheck && super.shouldRun(suiteName, testName) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index c78d8230e3a6..664cd37d1f7e 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} +import org.apache.spark.GlutenSortShuffleSuite import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite, GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite, GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuite, GlutenDataSourceV2Suite, GlutenFileDataSourceV2FallBackSuite, GlutenLocalScanSuite, GlutenSupportsCatalogOptionsSuite, GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite} @@ -229,6 +230,7 @@ class VeloxTestSettings extends BackendTestSettings { // Spark round UT for round(3.1415,3) is not correct. .exclude("round/bround") enableSuite[GlutenMathFunctionsSuite] + enableSuite[GlutenSortShuffleSuite] enableSuite[GlutenSortOrderExpressionsSuite] enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenStringExpressionsSuite] diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala new file mode 100644 index 000000000000..338d7992e38d --- /dev/null +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark + +class GlutenSortShuffleSuite extends SortShuffleSuite { + override def beforeAll(): Unit = { + super.beforeAll() + conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + } +} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 85f3f94cca95..bb782fde3d4d 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -74,6 +74,7 @@ class ClickHouseTestSettings extends BackendTestSettings { false // nativeDoValidate failed due to spark conf cleanup case "GlutenBloomFilterAggregateQuerySuite" => !bloomFilterCases.contains(testName) + case "GlutenSortShuffleSuite" => false case _ => true } preCheck && super.shouldRun(suiteName, testName) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 3b32cebca13f..57c1976221df 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} +import org.apache.spark.GlutenSortShuffleSuite import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{GlutenAnsiCastSuiteWithAnsiModeOff, GlutenAnsiCastSuiteWithAnsiModeOn, GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCastSuiteWithAnsiModeOn, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite, GlutenTryCastSuite} import org.apache.spark.sql.connector._ @@ -159,6 +160,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("random") .exclude("SPARK-9127 codegen with long seed") enableSuite[GlutenRegexpExpressionsSuite] + enableSuite[GlutenSortShuffleSuite] enableSuite[GlutenSortOrderExpressionsSuite] enableSuite[GlutenStringExpressionsSuite] enableSuite[VeloxAdaptiveQueryExecSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala new file mode 100644 index 000000000000..338d7992e38d --- /dev/null +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark + +class GlutenSortShuffleSuite extends SortShuffleSuite { + override def beforeAll(): Unit = { + super.beforeAll() + conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + } +} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 069d697bd454..7a38774273ac 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -76,6 +76,7 @@ class ClickHouseTestSettings extends BackendTestSettings { false // nativeDoValidate failed due to spark conf cleanup case "GlutenBloomFilterAggregateQuerySuite" => !bloomFilterCases.contains(testName) + case "GlutenSortShuffleSuite" => false case _ => true } preCheck && super.shouldRun(suiteName, testName) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 505417aebb50..6126844ade08 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} +import org.apache.spark.GlutenSortShuffleSuite import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} import org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite, GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite, GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter, GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite, GlutenDeleteFromTableSuite, GlutenDeltaBasedDeleteFromTableSuite, GlutenFileDataSourceV2FallBackSuite, GlutenGroupBasedDeleteFromTableSuite, GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, GlutenMetadataColumnSuite, GlutenSupportsCatalogOptionsSuite, GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite} @@ -137,6 +138,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("random") .exclude("SPARK-9127 codegen with long seed") enableSuite[GlutenRegexpExpressionsSuite] + enableSuite[GlutenSortShuffleSuite] enableSuite[GlutenSortOrderExpressionsSuite] enableSuite[GlutenStringExpressionsSuite] enableSuite[VeloxAdaptiveQueryExecSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala new file mode 100644 index 000000000000..338d7992e38d --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark + +class GlutenSortShuffleSuite extends SortShuffleSuite { + override def beforeAll(): Unit = { + super.beforeAll() + conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + } +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 069d697bd454..7a38774273ac 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -76,6 +76,7 @@ class ClickHouseTestSettings extends BackendTestSettings { false // nativeDoValidate failed due to spark conf cleanup case "GlutenBloomFilterAggregateQuerySuite" => !bloomFilterCases.contains(testName) + case "GlutenSortShuffleSuite" => false case _ => true } preCheck && super.shouldRun(suiteName, testName) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 27557f92046d..28f334878689 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} +import org.apache.spark.GlutenSortShuffleSuite import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} import org.apache.spark.sql.connector._ @@ -140,6 +141,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("random") .exclude("SPARK-9127 codegen with long seed") enableSuite[GlutenRegexpExpressionsSuite] + enableSuite[GlutenSortShuffleSuite] enableSuite[GlutenSortOrderExpressionsSuite] enableSuite[GlutenStringExpressionsSuite] enableSuite[VeloxAdaptiveQueryExecSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala new file mode 100644 index 000000000000..338d7992e38d --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark + +class GlutenSortShuffleSuite extends SortShuffleSuite { + override def beforeAll(): Unit = { + super.beforeAll() + conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + } +} From 1fd19a563b7ce1e2b0a0cb5a79a4044e743d3ab0 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 30 May 2024 11:00:06 +0800 Subject: [PATCH 173/402] [VL] Following #5889, correct / simplify the table indenting algorithm (#5917) --- .../integration/action/Parameterized.scala | 2 +- .../integration/action/TableRender.scala | 10 ++++++---- .../integration/action/TableRenderTest.scala | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala index 799b7632e02c..8f5bc0946643 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala @@ -227,7 +227,7 @@ case class TestResultLines( metricNames: Seq[String], lines: Iterable[TestResultLine]) { def print(): Unit = { - val fields = ListBuffer[String]("Query ID", "Succeed") + val fields = ListBuffer[String]("Query ID", "Succeeded") dimNames.foreach(dimName => fields.append(dimName)) metricNames.foreach(metricName => fields.append(metricName)) fields.append("Row Count") diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala index b25a5db93278..4cded2848b6e 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala @@ -116,12 +116,14 @@ object TableRender { def updateWidth(field: Field, lowerBound: Int): Unit = { field match { case branch @ Field.Branch(name, children) => - val childLowerBound = - Math.ceil((lowerBound max name.length + 2).toDouble / children.size.toDouble).toInt - children.foreach(child => updateWidth(child, childLowerBound)) + val leafLowerBound = + Math + .ceil((lowerBound max name.length + 2).toDouble / branch.leafs.size.toDouble) + .toInt + children.foreach(child => updateWidth(child, leafLowerBound * child.leafs.size)) val childrenWidth = children.map(child => widthMap(System.identityHashCode(child))).sum - val width = childLowerBound * children.size max childrenWidth + children.size - 1 + val width = childrenWidth + children.size - 1 val hash = System.identityHashCode(branch) widthMap += hash -> width case leaf @ Field.Leaf(name) => diff --git a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala index 87ad23f3622e..ce7b0974ce8b 100644 --- a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala +++ b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala @@ -82,10 +82,28 @@ object TableRenderTest { Console.out.println() } + def case4(): Unit = { + val render: TableRender[Seq[String]] = TableRender.create( + Branch( + "ABBBBBBBBBBBBBBBBBBBBBBBBBBBBC", + List(Branch("AB", List(Leaf("A"), Leaf("B"))), Leaf("C"))), + Branch("DE", List(Leaf("D"), Leaf("E"))))(new RowParser[Seq[String]] { + override def parse(rowFactory: FieldAppender.RowAppender, row: Seq[String]): Unit = { + val inc = rowFactory.incremental() + row.foreach(ceil => inc.next().write(ceil)) + } + }) + + render.appendRow(List("aaaa", "b", "cccccc", "d", "eeeee")) + render.print(Console.out) + Console.out.println() + } + def main(args: Array[String]): Unit = { case0() case1() case2() case3() + case4() } } From 2739490083354e68fe7627c1c9880eb266a25edf Mon Sep 17 00:00:00 2001 From: Yuan Date: Thu, 30 May 2024 12:07:02 +0800 Subject: [PATCH 174/402] [GLUTEN-4942][VL] refine vcpkg package script (#5900) refines the vcpkg script to allow use the GHA docker image to build package, with maven cache & ccache Signed-off-by: Yuan Zhou --- dev/package-vcpkg.sh | 5 +++-- dev/vcpkg/Makefile | 12 +++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/dev/package-vcpkg.sh b/dev/package-vcpkg.sh index 3f14f1c902d1..4a248b556726 100755 --- a/dev/package-vcpkg.sh +++ b/dev/package-vcpkg.sh @@ -6,9 +6,10 @@ CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) GLUTEN_DIR="$CURRENT_DIR/.." cd "$GLUTEN_DIR" -source ./dev/vcpkg/env.sh +source /opt/rh/devtoolset-9/enable +source /opt/gluten/dev/vcpkg/env.sh ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_hdfs=ON mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests -mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests \ No newline at end of file +mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests diff --git a/dev/vcpkg/Makefile b/dev/vcpkg/Makefile index e52393b040ad..11cd02d8819b 100644 --- a/dev/vcpkg/Makefile +++ b/dev/vcpkg/Makefile @@ -1,4 +1,4 @@ -DOCKER_IMAGE=gluten-builder-vcpkg +DOCKER_IMAGE=apache/gluten:gluten-vcpkg-builder_2024_05_22 GLUTEN_REPO=$(shell realpath -L ../..) CCACHE_DIR=$(HOME)/.ccache @@ -8,12 +8,14 @@ MAVEN_M2_DIR=$(HOME)/.m2 $(info $(GLUTEN_REPO)) .PHONY: docker-image build -build: docker-image | $(CCACHE_DIR) $(VCPKG_BINARY_CACHE_DIR) $(MAVEN_M2_DIR) +build: $(CCACHE_DIR) $(VCPKG_BINARY_CACHE_DIR) $(MAVEN_M2_DIR) docker run --rm -ti \ -v $(GLUTEN_REPO):$(GLUTEN_REPO) \ - -v $(VCPKG_BINARY_CACHE_DIR):/home/build/.cache/vcpkg \ - -v $(MAVEN_M2_DIR):/home/build/.m2 \ - -v $(CCACHE_DIR):/home/build/.ccache \ + -v $(VCPKG_BINARY_CACHE_DIR):/root/.cache/vcpkg \ + -v $(MAVEN_M2_DIR):/root/.m2 \ + -v $(CCACHE_DIR):/root/.ccache \ + -e http_proxy \ + -e https_proxy \ --workdir $(GLUTEN_REPO) \ -ti \ $(DOCKER_IMAGE) \ From d35d1dc5e4450fdf58b8092ea26a0c928de29a48 Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Thu, 30 May 2024 13:09:25 +0800 Subject: [PATCH 175/402] [CH] Adaptive sort memory controll and support memory sort shuffle (#5893) * optimize sort and shuffle * change block size config * support memory sort local shuffle * fix bug * support memory sort shuffle * update ch version * fix check style * fix bug * fix bug --------- Co-authored-by: liuneng1994 --- .../CHShuffleSplitterJniWrapper.java | 21 +- .../backendsapi/clickhouse/CHBackend.scala | 2 +- .../clickhouse/CHTransformerApi.scala | 6 - .../shuffle/CHColumnarShuffleWriter.scala | 10 +- ...useColumnarExternalSortShuffleSuite.scala} | 63 +--- ...kHouseColumnarMemorySortShuffleSuite.scala | 128 +++++++ ...utenClickHouseMergeTreeOptimizeSuite.scala | 20 +- cpp-ch/clickhouse.version | 2 +- cpp-ch/local-engine/Common/CHUtil.cpp | 21 ++ cpp-ch/local-engine/Common/CHUtil.h | 4 + cpp-ch/local-engine/Parser/SortRelParser.cpp | 9 +- .../Shuffle/CachedShuffleWriter.cpp | 79 ++-- .../Shuffle/CachedShuffleWriter.h | 15 +- .../local-engine/Shuffle/PartitionWriter.cpp | 338 +++++++++++++++--- cpp-ch/local-engine/Shuffle/PartitionWriter.h | 124 +++++-- .../local-engine/Shuffle/SelectorBuilder.cpp | 6 +- cpp-ch/local-engine/Shuffle/SelectorBuilder.h | 5 +- cpp-ch/local-engine/Shuffle/ShuffleSplitter.h | 3 +- .../Shuffle/SortedPartitionDataMerger.h | 2 + cpp-ch/local-engine/local_engine_jni.cpp | 15 +- ...lebornHashBasedColumnarShuffleWriter.scala | 4 +- ...RSSColumnarExternalSortShuffleSuite.scala} | 4 +- ...useRSSColumnarMemorySortShuffleSuite.scala | 136 +++++++ .../org/apache/gluten/GlutenConfig.scala | 31 +- 24 files changed, 824 insertions(+), 224 deletions(-) rename backends-clickhouse/src/test/scala/org/apache/gluten/execution/{GlutenClickHouseColumnarSortShuffleAQESuite.scala => GlutenClickHouseColumnarExternalSortShuffleSuite.scala} (52%) create mode 100644 backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarMemorySortShuffleSuite.scala rename gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/{GlutenClickHouseRSSColumnarSortShuffleAQESuite.scala => GlutenClickHouseRSSColumnarExternalSortShuffleSuite.scala} (97%) create mode 100644 gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarMemorySortShuffleSuite.scala diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java index f81ec88c254e..815bf472c027 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHShuffleSplitterJniWrapper.java @@ -37,7 +37,8 @@ public long make( boolean flushBlockBufferBeforeEvict, long maxSortBufferSize, boolean spillFirstlyBeforeStop, - boolean forceSort) { + boolean forceExternalSort, + boolean forceMemorySort) { return nativeMake( part.getShortName(), part.getNumPartitions(), @@ -57,7 +58,8 @@ public long make( flushBlockBufferBeforeEvict, maxSortBufferSize, spillFirstlyBeforeStop, - forceSort); + forceExternalSort, + forceMemorySort); } public long makeForRSS( @@ -70,7 +72,9 @@ public long makeForRSS( String hashAlgorithm, Object pusher, boolean throwIfMemoryExceed, - boolean flushBlockBufferBeforeEvict) { + boolean flushBlockBufferBeforeEvict, + boolean forceExternalSort, + boolean forceMemorySort) { return nativeMakeForRSS( part.getShortName(), part.getNumPartitions(), @@ -84,7 +88,9 @@ public long makeForRSS( hashAlgorithm, pusher, throwIfMemoryExceed, - flushBlockBufferBeforeEvict); + flushBlockBufferBeforeEvict, + forceExternalSort, + forceMemorySort); } public native long nativeMake( @@ -106,7 +112,8 @@ public native long nativeMake( boolean flushBlockBufferBeforeEvict, long maxSortBufferSize, boolean spillFirstlyBeforeStop, - boolean forceSort); + boolean forceSort, + boolean forceMemorySort); public native long nativeMakeForRSS( String shortName, @@ -121,7 +128,9 @@ public native long nativeMakeForRSS( String hashAlgorithm, Object pusher, boolean throwIfMemoryExceed, - boolean flushBlockBufferBeforeEvict); + boolean flushBlockBufferBeforeEvict, + boolean forceSort, + boolean forceMemorySort); public native void split(long splitterId, long block); diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index bc0c8d1c07f2..e5f68a8691ba 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -127,7 +127,7 @@ object CHBackendSettings extends BackendSettingsApi with Logging { val GLUTEN_MAX_SHUFFLE_READ_BYTES: String = GlutenConfig.GLUTEN_CONFIG_PREFIX + CHBackend.BACKEND_NAME + ".runtime_config.max_source_concatenate_bytes" - val GLUTEN_MAX_SHUFFLE_READ_BYTES_DEFAULT = -1 + val GLUTEN_MAX_SHUFFLE_READ_BYTES_DEFAULT = GLUTEN_MAX_BLOCK_SIZE_DEFAULT * 256 def affinityMode: String = { SparkEnv.get.conf diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index c75cf4788ba9..ea3398e77dfa 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -91,12 +91,6 @@ class CHTransformerApi extends TransformerApi with Logging { val offHeapSize = nativeConfMap.getOrDefault("spark.gluten.memory.offHeap.size.in.bytes", "0").toLong if (offHeapSize > 0) { - // Only set default max_bytes_before_external_sort for CH when it is not set explicitly. - val sortSpillKey = settingPrefix + "max_bytes_before_external_sort"; - if (!nativeConfMap.containsKey(sortSpillKey)) { - val sortSpillValue = offHeapSize * 0.5 - nativeConfMap.put(sortSpillKey, sortSpillValue.toLong.toString) - } // Only set default max_bytes_before_external_group_by for CH when it is not set explicitly. val groupBySpillKey = settingPrefix + "max_bytes_before_external_group_by"; diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala index 3a80e18bdcd6..4a1adbec7418 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala @@ -61,7 +61,8 @@ class CHColumnarShuffleWriter[K, V]( GlutenConfig.getConf.chColumnarFlushBlockBufferBeforeEvict private val maxSortBufferSize = GlutenConfig.getConf.chColumnarMaxSortBufferSize private val spillFirstlyBeforeStop = GlutenConfig.getConf.chColumnarSpillFirstlyBeforeStop - private val forceSortShuffle = GlutenConfig.getConf.chColumnarForceSortShuffle + private val forceExternalSortShuffle = GlutenConfig.getConf.chColumnarForceExternalSortShuffle + private val forceMemorySortShuffle = GlutenConfig.getConf.chColumnarForceMemorySortShuffle private val spillThreshold = GlutenConfig.getConf.chColumnarShuffleSpillThreshold private val jniWrapper = new CHShuffleSplitterJniWrapper // Are we in the process of stopping? Because map tasks can call stop() with success = true @@ -115,7 +116,8 @@ class CHColumnarShuffleWriter[K, V]( flushBlockBufferBeforeEvict, maxSortBufferSize, spillFirstlyBeforeStop, - forceSortShuffle + forceExternalSortShuffle, + forceMemorySortShuffle ) CHNativeMemoryAllocators.createSpillable( "ShuffleWriter", @@ -127,9 +129,9 @@ class CHColumnarShuffleWriter[K, V]( "is created. This behavior should be optimized by moving memory " + "allocations from make() to split()") } - logInfo(s"Gluten shuffle writer: Trying to spill $size bytes of data") + logError(s"Gluten shuffle writer: Trying to spill $size bytes of data") val spilled = splitterJniWrapper.evict(nativeSplitter); - logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data") + logError(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data") spilled } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarSortShuffleAQESuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarExternalSortShuffleSuite.scala similarity index 52% rename from backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarSortShuffleAQESuite.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarExternalSortShuffleSuite.scala index 098c7117bf7d..be36cd998485 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarSortShuffleAQESuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarExternalSortShuffleSuite.scala @@ -17,10 +17,9 @@ package org.apache.gluten.execution import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.CoalescedPartitionSpec -import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -class GlutenClickHouseColumnarSortShuffleAQESuite +class GlutenClickHouseColumnarExternalSortShuffleSuite extends GlutenClickHouseTPCHAbstractSuite with AdaptiveSparkPlanHelper { @@ -36,29 +35,11 @@ class GlutenClickHouseColumnarSortShuffleAQESuite .set("spark.sql.shuffle.partitions", "5") .set("spark.sql.autoBroadcastJoinThreshold", "10MB") .set("spark.sql.adaptive.enabled", "true") - .set("spark.gluten.sql.columnar.backend.ch.forceSortShuffle", "true") + .set("spark.gluten.sql.columnar.backend.ch.forceExternalSortShuffle", "true") } test("TPCH Q1") { - runTPCHQuery(1) { - df => - assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec]) - - val colCustomShuffleReaderExecs = collect(df.queryExecution.executedPlan) { - case csr: AQEShuffleReadExec => csr - } - assert(colCustomShuffleReaderExecs.size == 2) - val coalescedPartitionSpec0 = colCustomShuffleReaderExecs(0) - .partitionSpecs(0) - .asInstanceOf[CoalescedPartitionSpec] - assert(coalescedPartitionSpec0.startReducerIndex == 0) - assert(coalescedPartitionSpec0.endReducerIndex == 5) - val coalescedPartitionSpec1 = colCustomShuffleReaderExecs(1) - .partitionSpecs(0) - .asInstanceOf[CoalescedPartitionSpec] - assert(coalescedPartitionSpec1.startReducerIndex == 0) - assert(coalescedPartitionSpec1.endReducerIndex == 5) - } + runTPCHQuery(1) { df => } } test("TPCH Q2") { @@ -98,14 +79,7 @@ class GlutenClickHouseColumnarSortShuffleAQESuite } test("TPCH Q11") { - runTPCHQuery(11) { - df => - assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec]) - val adaptiveSparkPlanExec = collectWithSubqueries(df.queryExecution.executedPlan) { - case adaptive: AdaptiveSparkPlanExec => adaptive - } - assert(adaptiveSparkPlanExec.size == 2) - } + runTPCHQuery(11) { df => } } test("TPCH Q12") { @@ -121,14 +95,7 @@ class GlutenClickHouseColumnarSortShuffleAQESuite } test("TPCH Q15") { - runTPCHQuery(15) { - df => - assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec]) - val adaptiveSparkPlanExec = collectWithSubqueries(df.queryExecution.executedPlan) { - case adaptive: AdaptiveSparkPlanExec => adaptive - } - assert(adaptiveSparkPlanExec.size == 2) - } + runTPCHQuery(15) { df => } } test("TPCH Q16") { @@ -140,13 +107,7 @@ class GlutenClickHouseColumnarSortShuffleAQESuite } test("TPCH Q18") { - runTPCHQuery(18) { - df => - val hashAggregates = collect(df.queryExecution.executedPlan) { - case hash: HashAggregateExecBaseTransformer => hash - } - assert(hashAggregates.size == 3) - } + runTPCHQuery(18) { df => } } test("TPCH Q19") { @@ -162,14 +123,6 @@ class GlutenClickHouseColumnarSortShuffleAQESuite } test("TPCH Q22") { - runTPCHQuery(22) { - df => - assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec]) - val adaptiveSparkPlanExec = collectWithSubqueries(df.queryExecution.executedPlan) { - case adaptive: AdaptiveSparkPlanExec => adaptive - } - assert(adaptiveSparkPlanExec.size == 3) - assert(adaptiveSparkPlanExec(1) == adaptiveSparkPlanExec(2)) - } + runTPCHQuery(22) { df => } } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarMemorySortShuffleSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarMemorySortShuffleSuite.scala new file mode 100644 index 000000000000..b9d580c7249c --- /dev/null +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseColumnarMemorySortShuffleSuite.scala @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.SparkConf +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + +class GlutenClickHouseColumnarMemorySortShuffleSuite + extends GlutenClickHouseTPCHAbstractSuite + with AdaptiveSparkPlanHelper { + + override protected val tablesPath: String = basePath + "/tpch-data-ch" + override protected val tpchQueries: String = rootPath + "queries/tpch-queries-ch" + override protected val queriesResults: String = rootPath + "mergetree-queries-output" + + /** Run Gluten + ClickHouse Backend with ColumnarShuffleManager */ + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + .set("spark.io.compression.codec", "LZ4") + .set("spark.sql.shuffle.partitions", "5") + .set("spark.sql.autoBroadcastJoinThreshold", "10MB") + .set("spark.sql.adaptive.enabled", "true") + .set("spark.gluten.sql.columnar.backend.ch.forceMemorySortShuffle", "true") + } + + test("TPCH Q1") { + runTPCHQuery(1) { df => } + } + + test("TPCH Q2") { + runTPCHQuery(2) { df => } + } + + test("TPCH Q3") { + runTPCHQuery(3) { df => } + } + + test("TPCH Q4") { + runTPCHQuery(4) { df => } + } + + test("TPCH Q5") { + runTPCHQuery(5) { df => } + } + + test("TPCH Q6") { + runTPCHQuery(6) { df => } + } + + test("TPCH Q7") { + runTPCHQuery(7) { df => } + } + + test("TPCH Q8") { + runTPCHQuery(8) { df => } + } + + test("TPCH Q9") { + runTPCHQuery(9) { df => } + } + + test("TPCH Q10") { + runTPCHQuery(10) { df => } + } + + test("TPCH Q11") { + runTPCHQuery(11) { df => } + } + + test("TPCH Q12") { + runTPCHQuery(12) { df => } + } + + test("TPCH Q13") { + runTPCHQuery(13) { df => } + } + + test("TPCH Q14") { + runTPCHQuery(14) { df => } + } + + test("TPCH Q15") { + runTPCHQuery(15) { df => } + } + + test("TPCH Q16") { + runTPCHQuery(16, noFallBack = false) { df => } + } + + test("TPCH Q17") { + runTPCHQuery(17) { df => } + } + + test("TPCH Q18") { + runTPCHQuery(18) { df => } + } + + test("TPCH Q19") { + runTPCHQuery(19) { df => } + } + + test("TPCH Q20") { + runTPCHQuery(20) { df => } + } + + test("TPCH Q21") { + runTPCHQuery(21, noFallBack = false) { df => } + } + + test("TPCH Q22") { + runTPCHQuery(22) { df => } + } +} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index d4302193f5b8..f016f9dc5d14 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -157,9 +157,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 812) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 372) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 232) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 239) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") // the second VACUUM will remove some empty folders assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 220) @@ -188,11 +188,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 398) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 286) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 306) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 270) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 276) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() @@ -219,11 +219,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 398) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 286) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 306) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 270) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 276) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() @@ -313,12 +313,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 931 else 1014 + if (sparkVersion.equals("3.2")) 499 else 528 }) spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 439 else 445 + if (sparkVersion.equals("3.2")) 315 else 321 }) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index b5d3aac8b42a..5125aabe536d 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence CH_BRANCH=rebase_ch/20240527 -CH_COMMIT=dd16f9435bf +CH_COMMIT=55b10ba376274f2a61a4c1daf1a2fb744155bd32 diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 9e2ce6304718..317adda2ae3e 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -555,6 +555,12 @@ DB::Context::ConfigurationPtr BackendInitializerUtil::initConfig(std::mapsetString(key.substr(CH_RUNTIME_CONFIG_PREFIX.size()), value); } } + + if (backend_conf_map.contains(GLUTEN_TASK_OFFHEAP)) + { + config->setString(CH_TASK_MEMORY, backend_conf_map.at(GLUTEN_TASK_OFFHEAP)); + } + return config; } @@ -672,6 +678,21 @@ void BackendInitializerUtil::initSettings(std::map & b settings.set("function_json_value_return_type_allow_complex", true); settings.set("function_json_value_return_type_allow_nullable", true); settings.set("precise_float_parsing", true); + if (backend_conf_map.contains(GLUTEN_TASK_OFFHEAP)) + { + auto task_memory = std::stoull(backend_conf_map.at(GLUTEN_TASK_OFFHEAP)); + if (!backend_conf_map.contains(CH_RUNTIME_SETTINGS_PREFIX + "max_bytes_before_external_sort")) + { + settings.max_bytes_before_external_sort = static_cast(0.8 * task_memory); + } + if (!backend_conf_map.contains(CH_RUNTIME_SETTINGS_PREFIX + "prefer_external_sort_block_bytes")) + { + auto mem_gb = task_memory / static_cast(1_GiB); + // 2.8x+5, Heuristics calculate the block size of external sort, [8,16] + settings.prefer_external_sort_block_bytes = std::max(std::min( + static_cast(2.8*mem_gb + 5), 16ul), 8ul) * 1024 * 1024; + } + } } void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index edbd91c50d22..458eec9d3ee9 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -1,3 +1,4 @@ +/* /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -168,6 +169,9 @@ class BackendInitializerUtil inline static const std::string S3A_PREFIX = "fs.s3a."; inline static const std::string SPARK_DELTA_PREFIX = "spark.databricks.delta."; + inline static const String GLUTEN_TASK_OFFHEAP = "spark.gluten.memory.task.offHeap.size.in.bytes"; + inline static const String CH_TASK_MEMORY = "off_heap_per_task"; + /// On yarn mode, native writing on hdfs cluster takes yarn container user as the user passed to libhdfs3, which /// will cause permission issue because yarn container user is not the owner of the hdfs dir to be written. /// So we need to get the spark user from env and pass it to libhdfs3. diff --git a/cpp-ch/local-engine/Parser/SortRelParser.cpp b/cpp-ch/local-engine/Parser/SortRelParser.cpp index 88141d030672..ea29e72d1324 100644 --- a/cpp-ch/local-engine/Parser/SortRelParser.cpp +++ b/cpp-ch/local-engine/Parser/SortRelParser.cpp @@ -40,8 +40,15 @@ SortRelParser::parse(DB::QueryPlanPtr query_plan, const substrait::Rel & rel, st size_t limit = parseLimit(rel_stack_); const auto & sort_rel = rel.sort(); auto sort_descr = parseSortDescription(sort_rel.sorts(), query_plan->getCurrentDataStream().header); + SortingStep::Settings settings(*getContext()); + size_t offheap_per_task = getContext()->getConfigRef().getUInt64("off_heap_per_task"); + double spill_mem_ratio = getContext()->getConfigRef().getDouble("spill_mem_ratio", 0.9); + settings.worth_external_sort = [offheap_per_task, spill_mem_ratio]() -> bool + { + return CurrentMemoryTracker::current_memory() > offheap_per_task * spill_mem_ratio; + }; auto sorting_step = std::make_unique( - query_plan->getCurrentDataStream(), sort_descr, limit, SortingStep::Settings(*getContext()), false); + query_plan->getCurrentDataStream(), sort_descr, limit, settings, false); sorting_step->setStepDescription("Sorting step"); steps.emplace_back(sorting_step.get()); query_plan->addStep(std::move(sorting_step)); diff --git a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp index 5a8629434172..559d9031862e 100644 --- a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp +++ b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp @@ -25,25 +25,23 @@ namespace DB { - namespace ErrorCodes { - extern const int BAD_ARGUMENTS; +extern const int BAD_ARGUMENTS; } } namespace local_engine { - using namespace DB; -CachedShuffleWriter::CachedShuffleWriter(const String & short_name, const SplitOptions & options_, jobject rss_pusher) : options(options_) +CachedShuffleWriter::CachedShuffleWriter(const String & short_name, const SplitOptions & options_, jobject rss_pusher) + : options(options_) { - bool use_external_sort_shuffle = (options.force_sort) && !rss_pusher; if (short_name == "rr") { - partitioner = std::make_unique(options.partition_num, use_external_sort_shuffle); + partitioner = std::make_unique(options.partition_num); } else if (short_name == "hash") { @@ -53,15 +51,15 @@ CachedShuffleWriter::CachedShuffleWriter(const String & short_name, const SplitO { hash_fields.push_back(std::stoi(expr)); } - partitioner = std::make_unique(options.partition_num, hash_fields, options_.hash_algorithm, use_external_sort_shuffle); + partitioner = std::make_unique(options.partition_num, hash_fields, options_.hash_algorithm); } else if (short_name == "single") { options.partition_num = 1; - partitioner = std::make_unique(options.partition_num, use_external_sort_shuffle); + partitioner = std::make_unique(options.partition_num); } else if (short_name == "range") - partitioner = std::make_unique(options.hash_exprs, options.partition_num, use_external_sort_shuffle); + partitioner = std::make_unique(options.hash_exprs, options.partition_num); else throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "unsupported splitter {}", short_name); @@ -77,32 +75,17 @@ CachedShuffleWriter::CachedShuffleWriter(const String & short_name, const SplitO jmethodID celeborn_push_partition_data_method = GetMethodID(env, celeborn_partition_pusher_class, "pushPartitionData", "(I[BI)I"); CLEAN_JNIENV - auto celeborn_client = std::make_unique(rss_pusher, celeborn_push_partition_data_method); - if (use_external_sort_shuffle) - { - partition_writer = std::make_unique(this, std::move(celeborn_client)); - sort_shuffle = true; - } - else - partition_writer = std::make_unique(this, std::move(celeborn_client)); - } - else - { - if (use_external_sort_shuffle) - { - partition_writer = std::make_unique(this); - sort_shuffle = true; - } - else - partition_writer = std::make_unique(this); + celeborn_client = std::make_unique(rss_pusher, celeborn_push_partition_data_method); } + split_result.partition_lengths.resize(options.partition_num, 0); split_result.raw_partition_lengths.resize(options.partition_num, 0); } void CachedShuffleWriter::split(DB::Block & block) { + lazyInitPartitionWriter(block); auto block_info = block.info; initOutputIfNeeded(block); @@ -145,18 +128,50 @@ void CachedShuffleWriter::initOutputIfNeeded(Block & block) } } -SplitResult CachedShuffleWriter::stop() +void CachedShuffleWriter::lazyInitPartitionWriter(Block & input_sample) { - partition_writer->stop(); + if (partition_writer) + return; + + auto avg_row_size = input_sample.allocatedBytes() / input_sample.rows(); + auto overhead_memory = std::max(avg_row_size, input_sample.columns() * 16) * options.split_size * options.partition_num; + auto use_sort_shuffle = overhead_memory > options.spill_threshold * 0.5 || options.partition_num >= 300; + auto use_external_sort_shuffle = options.force_external_sort; + auto use_memory_sort_shuffle = options.force_mermory_sort || use_sort_shuffle; + sort_shuffle = use_memory_sort_shuffle || use_external_sort_shuffle; + if (celeborn_client) + { + if (use_external_sort_shuffle) + partition_writer = std::make_unique(this, std::move(celeborn_client)); + else if (use_memory_sort_shuffle) + partition_writer = std::make_unique(this, std::move(celeborn_client)); + else + partition_writer = std::make_unique(this, std::move(celeborn_client)); + } + else + { + if (use_external_sort_shuffle) + partition_writer = std::make_unique(this); + else if (use_memory_sort_shuffle) + partition_writer = std::make_unique(this); + else + partition_writer = std::make_unique(this); + } + partitioner->setUseSortShuffle(sort_shuffle); + LOG_INFO(logger, "Use Partition Writer {}", partition_writer->getName()); +} - static auto * logger = &Poco::Logger::get("CachedShuffleWriter"); +SplitResult CachedShuffleWriter::stop() +{ + if (partition_writer) + partition_writer->stop(); LOG_INFO(logger, "CachedShuffleWriter stop, split result: {}", split_result.toString()); return split_result; } size_t CachedShuffleWriter::evictPartitions() { + if (!partition_writer) return 0; return partition_writer->evictPartitions(true, options.flush_block_buffer_before_evict); } - -} +} \ No newline at end of file diff --git a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.h b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.h index d1dd4ff2fce6..e6395c8e4712 100644 --- a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.h +++ b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.h @@ -24,10 +24,10 @@ namespace local_engine { - -class PartitionWriter; -class LocalPartitionWriter; -class CelebornPartitionWriter; + class CelebornClient; + class PartitionWriter; + class LocalPartitionWriter; + class CelebornPartitionWriter; class CachedShuffleWriter : public ShuffleWriterBase { @@ -35,8 +35,12 @@ class CachedShuffleWriter : public ShuffleWriterBase friend class PartitionWriter; friend class LocalPartitionWriter; friend class CelebornPartitionWriter; + friend class SortBasedPartitionWriter; + friend class MemorySortLocalPartitionWriter; + friend class MemorySortCelebornPartitionWriter; friend class ExternalSortLocalPartitionWriter; friend class ExternalSortCelebornPartitionWriter; + friend class Spillable; explicit CachedShuffleWriter(const String & short_name, const SplitOptions & options, jobject rss_pusher = nullptr); ~CachedShuffleWriter() override = default; @@ -47,6 +51,7 @@ class CachedShuffleWriter : public ShuffleWriterBase private: void initOutputIfNeeded(DB::Block & block); + void lazyInitPartitionWriter(DB::Block & input_sample); bool stopped = false; DB::Block output_header; @@ -55,7 +60,9 @@ class CachedShuffleWriter : public ShuffleWriterBase std::unique_ptr partitioner; std::vector output_columns_indicies; std::unique_ptr partition_writer; + std::unique_ptr celeborn_client; bool sort_shuffle = false; + Poco::Logger* logger = &Poco::Logger::get("CachedShuffleWriter"); }; } diff --git a/cpp-ch/local-engine/Shuffle/PartitionWriter.cpp b/cpp-ch/local-engine/Shuffle/PartitionWriter.cpp index e0b69316de6a..d02c79e0a5d6 100644 --- a/cpp-ch/local-engine/Shuffle/PartitionWriter.cpp +++ b/cpp-ch/local-engine/Shuffle/PartitionWriter.cpp @@ -49,6 +49,7 @@ extern const int LOGICAL_ERROR; using namespace DB; namespace local_engine { +static const String PARTITION_COLUMN_NAME = "partition"; void PartitionWriter::write(const PartitionInfo & partition_info, DB::Block & block) { @@ -120,7 +121,7 @@ void PartitionWriter::write(const PartitionInfo & partition_info, DB::Block & bl } /// Only works for local partition writer - if (!supportsEvictSinglePartition() && options->spill_threshold && current_cached_bytes >= options->spill_threshold) + if (!supportsEvictSinglePartition() && options->spill_threshold && CurrentMemoryTracker::current_memory() >= options->spill_threshold) unsafeEvictPartitions(false, options->flush_block_buffer_before_evict); shuffle_writer->split_result.total_split_time += watch.elapsedNanoseconds(); @@ -157,20 +158,18 @@ size_t LocalPartitionWriter::unsafeEvictPartitions(bool for_memory_spill, bool f if (buffer->empty()) continue; - PartitionSpillInfo partition_spill_info; - partition_spill_info.start = output.count(); + std::pair offsets; + offsets.first = output.count(); spilled_bytes += buffer->bytes(); size_t written_bytes = buffer->spill(writer); res += written_bytes; compressed_output.sync(); - partition_spill_info.length = output.count() - partition_spill_info.start; + offsets.second = output.count() - offsets.first; shuffle_writer->split_result.raw_partition_lengths[partition_id] += written_bytes; - partition_spill_info.partition_id = partition_id; - info.partition_spill_infos.emplace_back(partition_spill_info); + info.partition_spill_infos[partition_id] = offsets; } - spill_infos.emplace_back(info); shuffle_writer->split_result.total_compress_time += compressed_output.getCompressTime(); shuffle_writer->split_result.total_write_time += compressed_output.getWriteTime(); @@ -182,8 +181,7 @@ size_t LocalPartitionWriter::unsafeEvictPartitions(bool for_memory_spill, bool f { // escape memory track from current thread status; add untracked memory limit for create thread object, avoid trigger memory spill again IgnoreMemoryTracker ignore(settings.spill_memory_overhead); - ThreadFromGlobalPool thread(spill_to_file); - thread.join(); + spill_to_file(); } else { @@ -194,20 +192,35 @@ size_t LocalPartitionWriter::unsafeEvictPartitions(bool for_memory_spill, bool f return res; } -std::vector LocalPartitionWriter::mergeSpills(WriteBuffer & data_file) +String Spillable::getNextSpillFile() +{ + auto file_name = std::to_string(split_options.shuffle_id) + "_" + std::to_string(split_options.map_id) + "_" + std::to_string(spill_infos.size()); + std::hash hasher; + auto hash = hasher(file_name); + auto dir_id = hash % split_options.local_dirs_list.size(); + auto sub_dir_id = (hash / split_options.local_dirs_list.size()) % split_options.num_sub_dirs; + + std::string dir = std::filesystem::path(split_options.local_dirs_list[dir_id]) / std::format("{:02x}", sub_dir_id); + if (!std::filesystem::exists(dir)) + std::filesystem::create_directories(dir); + return std::filesystem::path(dir) / file_name; +} + +std::vector Spillable::mergeSpills(CachedShuffleWriter * shuffle_writer, WriteBuffer & data_file, ExtraData extra_data) { auto codec = DB::CompressionCodecFactory::instance().get(boost::to_upper_copy(shuffle_writer->options.compress_method), {}); + CompressedWriteBuffer compressed_output(data_file, codec, shuffle_writer->options.io_buffer_size); NativeWriter writer(compressed_output, shuffle_writer->output_header); std::vector partition_length(shuffle_writer->options.partition_num, 0); - std::vector spill_inputs; + std::vector> spill_inputs; spill_inputs.reserve(spill_infos.size()); for (const auto & spill : spill_infos) { // only use readBig - spill_inputs.emplace_back(std::make_shared(spill.spilled_file, 0)); + spill_inputs.emplace_back(std::make_shared(spill.spilled_file, 0)); } Stopwatch write_time_watch; @@ -215,33 +228,46 @@ std::vector LocalPartitionWriter::mergeSpills(WriteBuffer & data_file) Stopwatch serialization_time_watch; size_t merge_io_time = 0; String buffer; - for (size_t partition_id = 0; partition_id < partition_block_buffer.size(); ++partition_id) + for (size_t partition_id = 0; partition_id < split_options.partition_num; ++partition_id) { auto size_before = data_file.count(); io_time_watch.restart(); for (size_t i = 0; i < spill_infos.size(); ++i) { - size_t size = spill_infos[i].partition_spill_infos[partition_id].length; + if (!spill_infos[i].partition_spill_infos.contains(partition_id)) + { + continue; + } + size_t size = spill_infos[i].partition_spill_infos[partition_id].second; + size_t offset = spill_infos[i].partition_spill_infos[partition_id].first; + if (!size) + { + continue; + } buffer.reserve(size); - auto count = spill_inputs[i]->readBig(buffer.data(), size); + auto count = spill_inputs[i]->readBigAt(buffer.data(), size, offset, nullptr); + + chassert(count == size); data_file.write(buffer.data(), count); } merge_io_time += io_time_watch.elapsedNanoseconds(); serialization_time_watch.restart(); - if (!partition_block_buffer[partition_id]->empty()) + if (!extra_data.partition_block_buffer.empty() && !extra_data.partition_block_buffer[partition_id]->empty()) { - Block block = partition_block_buffer[partition_id]->releaseColumns(); - partition_buffer[partition_id]->addBlock(std::move(block)); + Block block = extra_data.partition_block_buffer[partition_id]->releaseColumns(); + extra_data.partition_buffer[partition_id]->addBlock(std::move(block)); + } + if (!extra_data.partition_buffer.empty()) + { + size_t raw_size = extra_data.partition_buffer[partition_id]->spill(writer); + shuffle_writer->split_result.raw_partition_lengths[partition_id] += raw_size; } - size_t raw_size = partition_buffer[partition_id]->spill(writer); - compressed_output.sync(); partition_length[partition_id] = data_file.count() - size_before; shuffle_writer->split_result.total_serialize_time += serialization_time_watch.elapsedNanoseconds(); shuffle_writer->split_result.total_bytes_written += partition_length[partition_id]; - shuffle_writer->split_result.raw_partition_lengths[partition_id] += raw_size; } shuffle_writer->split_result.total_write_time += write_time_watch.elapsedNanoseconds(); @@ -253,32 +279,43 @@ std::vector LocalPartitionWriter::mergeSpills(WriteBuffer & data_file) for (const auto & spill : spill_infos) std::filesystem::remove(spill.spilled_file); - return partition_length; } -LocalPartitionWriter::LocalPartitionWriter(CachedShuffleWriter * shuffle_writer_) : PartitionWriter(shuffle_writer_) +void SortBasedPartitionWriter::write(const PartitionInfo & info, DB::Block & block) { + Stopwatch write_time_watch; + if (output_header.columns() == 0) + output_header = block.cloneEmpty(); + auto partition_column = ColumnUInt64::create(); + partition_column->reserve(block.rows()); + partition_column->getData().insert_assume_reserved(info.src_partition_num.begin(), info.src_partition_num.end()); + block.insert({std::move(partition_column), std::make_shared(), PARTITION_COLUMN_NAME}); + if (sort_header.columns() == 0) + { + sort_header = block.cloneEmpty(); + sort_description.emplace_back(SortColumnDescription(PARTITION_COLUMN_NAME)); + } + // partial sort + sortBlock(block, sort_description); + Chunk chunk; + chunk.setColumns(block.getColumns(), block.rows()); + accumulated_blocks.emplace_back(std::move(chunk)); + current_accumulated_bytes += accumulated_blocks.back().allocatedBytes(); + current_accumulated_rows += accumulated_blocks.back().getNumRows(); + shuffle_writer->split_result.total_write_time += write_time_watch.elapsedNanoseconds(); + if (options->spill_threshold && CurrentMemoryTracker::current_memory() >= options->spill_threshold) + unsafeEvictPartitions(false, false); } -String LocalPartitionWriter::getNextSpillFile() +LocalPartitionWriter::LocalPartitionWriter(CachedShuffleWriter * shuffle_writer_) : PartitionWriter(shuffle_writer_), Spillable(shuffle_writer_->options) { - auto file_name = std::to_string(options->shuffle_id) + "_" + std::to_string(options->map_id) + "_" + std::to_string(spill_infos.size()); - std::hash hasher; - auto hash = hasher(file_name); - auto dir_id = hash % options->local_dirs_list.size(); - auto sub_dir_id = (hash / options->local_dirs_list.size()) % options->num_sub_dirs; - - std::string dir = std::filesystem::path(options->local_dirs_list[dir_id]) / std::format("{:02x}", sub_dir_id); - if (!std::filesystem::exists(dir)) - std::filesystem::create_directories(dir); - return std::filesystem::path(dir) / file_name; } void LocalPartitionWriter::unsafeStop() { WriteBufferFromFile output(options->data_file, options->io_buffer_size); - auto offsets = mergeSpills(output); + auto offsets = mergeSpills(shuffle_writer, output, {partition_block_buffer, partition_buffer}); shuffle_writer->split_result.partition_lengths = offsets; } @@ -335,30 +372,211 @@ size_t PartitionWriter::bytes() const return bytes; } -void ExternalSortLocalPartitionWriter::write(const PartitionInfo & info, DB::Block & block) +size_t MemorySortLocalPartitionWriter::unsafeEvictPartitions(bool for_memory_spill, bool /*flush_block_buffer*/) { - Stopwatch write_time_watch; - if (output_header.columns() == 0) - output_header = block.cloneEmpty(); - static const String partition_column_name = "partition"; - auto partition_column = ColumnUInt64::create(); - partition_column->reserve(block.rows()); - partition_column->getData().insert_assume_reserved(info.src_partition_num.begin(), info.src_partition_num.end()); - block.insert({std::move(partition_column), std::make_shared(), partition_column_name}); - if (sort_header.columns() == 0) + size_t res = 0; + size_t spilled_bytes = 0; + + auto spill_to_file = [this, &res, &spilled_bytes]() { - sort_header = block.cloneEmpty(); - sort_description.emplace_back(SortColumnDescription(partition_column_name)); + if (accumulated_blocks.empty()) + return; + auto file = getNextSpillFile(); + WriteBufferFromFile output(file, shuffle_writer->options.io_buffer_size); + auto codec = DB::CompressionCodecFactory::instance().get(boost::to_upper_copy(shuffle_writer->options.compress_method), {}); + CompressedWriteBuffer compressed_output(output, codec, shuffle_writer->options.io_buffer_size); + NativeWriter writer(compressed_output, output_header); + + SpillInfo info; + info.spilled_file = file; + + Stopwatch serialization_time_watch; + MergeSorter sorter(sort_header, std::move(accumulated_blocks), sort_description, adaptiveBlockSize(), 0); + size_t cur_partition_id = 0; + info.partition_spill_infos[cur_partition_id] = {0,0}; + while (auto data = sorter.read()) + { + Block serialized_block = sort_header.cloneWithColumns(data.detachColumns()); + const auto partitions = serialized_block.getByName(PARTITION_COLUMN_NAME).column; + serialized_block.erase(PARTITION_COLUMN_NAME); + size_t row_offset = 0; + while (row_offset < serialized_block.rows()) + { + auto last_idx = searchLastPartitionIdIndex(partitions, row_offset, cur_partition_id); + if (last_idx < 0) + { + auto& last = info.partition_spill_infos[cur_partition_id]; + compressed_output.sync(); + last.second = output.count() - last.first; + cur_partition_id++; + info.partition_spill_infos[cur_partition_id] = {last.first + last.second, 0}; + continue; + } + + if (row_offset == 0 && last_idx == serialized_block.rows() - 1) + { + auto count = writer.write(serialized_block); + shuffle_writer->split_result.raw_partition_lengths[cur_partition_id] += count; + break; + } + else + { + auto cut_block = serialized_block.cloneWithCutColumns(row_offset, last_idx - row_offset + 1); + + auto count = writer.write(cut_block); + shuffle_writer->split_result.raw_partition_lengths[cur_partition_id] += count; + row_offset = last_idx + 1; + if (last_idx != serialized_block.rows() - 1) + { + auto& last = info.partition_spill_infos[cur_partition_id]; + compressed_output.sync(); + last.second = output.count() - last.first; + cur_partition_id++; + info.partition_spill_infos[cur_partition_id] = {last.first + last.second, 0}; + } + } + } + } + compressed_output.sync(); + auto& last = info.partition_spill_infos[cur_partition_id]; + last.second = output.count() - last.first; + spilled_bytes = current_accumulated_bytes; + res = current_accumulated_bytes; + current_accumulated_bytes = 0; + current_accumulated_rows = 0; + std::erase_if(info.partition_spill_infos, [](const auto & item) + { + auto const& [key, value] = item; + return value.second == 0; + }); + spill_infos.emplace_back(info); + shuffle_writer->split_result.total_compress_time += compressed_output.getCompressTime(); + shuffle_writer->split_result.total_io_time += compressed_output.getWriteTime(); + shuffle_writer->split_result.total_serialize_time += serialization_time_watch.elapsedNanoseconds(); + }; + + Stopwatch spill_time_watch; + if (for_memory_spill && options->throw_if_memory_exceed) + { + // escape memory track from current thread status; add untracked memory limit for create thread object, avoid trigger memory spill again + IgnoreMemoryTracker ignore(settings.spill_memory_overhead); + spill_to_file(); } - // partial sort - sortBlock(block, sort_description); - Chunk chunk; - chunk.setColumns(block.getColumns(), block.rows()); - accumulated_blocks.emplace_back(std::move(chunk)); - current_accumulated_bytes += accumulated_blocks.back().allocatedBytes(); - if (current_accumulated_bytes >= max_sort_buffer_size) - unsafeEvictPartitions(false, false); - shuffle_writer->split_result.total_write_time += write_time_watch.elapsedNanoseconds(); + else + { + spill_to_file(); + } + shuffle_writer->split_result.total_spill_time += spill_time_watch.elapsedNanoseconds(); + shuffle_writer->split_result.total_bytes_spilled += spilled_bytes; + return res; +} + +void MemorySortLocalPartitionWriter::unsafeStop() +{ + unsafeEvictPartitions(false, false); + WriteBufferFromFile output(options->data_file, options->io_buffer_size); + auto offsets = mergeSpills(shuffle_writer, output); + shuffle_writer->split_result.partition_lengths = offsets; +} + +size_t MemorySortCelebornPartitionWriter::unsafeEvictPartitions(bool for_memory_spill, bool flush_block_buffer) +{ + size_t res = 0; + size_t spilled_bytes = 0; + auto spill_to_celeborn = [this, for_memory_spill, flush_block_buffer, &res, &spilled_bytes]() + { + Stopwatch serialization_time_watch; + + /// Skip empty buffer + if (accumulated_blocks.empty()) + return; + + WriteBufferFromOwnString output; + auto codec = DB::CompressionCodecFactory::instance().get(boost::to_upper_copy(shuffle_writer->options.compress_method), {}); + CompressedWriteBuffer compressed_output(output, codec, shuffle_writer->options.io_buffer_size); + NativeWriter writer(compressed_output, shuffle_writer->output_header); + + MergeSorter sorter(sort_header, std::move(accumulated_blocks), sort_description, adaptiveBlockSize(), 0); + size_t cur_partition_id = 0; + auto push_to_celeborn = [&]() + { + compressed_output.sync(); + auto& data = output.str(); + if (!data.empty()) + { + Stopwatch push_time_watch; + celeborn_client->pushPartitionData(cur_partition_id, data.data(), data.size()); + shuffle_writer->split_result.total_io_time += push_time_watch.elapsedNanoseconds(); + shuffle_writer->split_result.partition_lengths[cur_partition_id] += data.size(); + } + output.restart(); + }; + + while (auto data = sorter.read()) + { + Block serialized_block = sort_header.cloneWithColumns(data.detachColumns()); + const auto partitions = serialized_block.getByName(PARTITION_COLUMN_NAME).column; + serialized_block.erase(PARTITION_COLUMN_NAME); + size_t row_offset = 0; + while (row_offset < serialized_block.rows()) + { + auto last_idx = searchLastPartitionIdIndex(partitions, row_offset, cur_partition_id); + if (last_idx < 0) + { + push_to_celeborn(); + cur_partition_id++; + continue; + } + + if (row_offset == 0 && last_idx == serialized_block.rows() - 1) + { + auto count = writer.write(serialized_block); + shuffle_writer->split_result.raw_partition_lengths[cur_partition_id] += count; + break; + } + auto cut_block = serialized_block.cloneWithCutColumns(row_offset, last_idx - row_offset + 1); + auto count = writer.write(cut_block); + shuffle_writer->split_result.raw_partition_lengths[cur_partition_id] += count; + row_offset = last_idx + 1; + if (last_idx != serialized_block.rows() - 1) + { + push_to_celeborn(); + cur_partition_id++; + } + } + } + push_to_celeborn(); + spilled_bytes = current_accumulated_bytes; + res = current_accumulated_bytes; + current_accumulated_bytes = 0; + current_accumulated_rows = 0; + + shuffle_writer->split_result.total_compress_time += compressed_output.getCompressTime(); + shuffle_writer->split_result.total_io_time += compressed_output.getWriteTime(); + + shuffle_writer->split_result.total_serialize_time += serialization_time_watch.elapsedNanoseconds(); + }; + + Stopwatch spill_time_watch; + if (for_memory_spill && options->throw_if_memory_exceed) + { + // escape memory track from current thread status; add untracked memory limit for create thread object, avoid trigger memory spill again + IgnoreMemoryTracker ignore(settings.spill_memory_overhead); + spill_to_celeborn(); + } + else + { + spill_to_celeborn(); + } + + shuffle_writer->split_result.total_spill_time += spill_time_watch.elapsedNanoseconds(); + shuffle_writer->split_result.total_bytes_spilled += spilled_bytes; + return res; +} + +void MemorySortCelebornPartitionWriter::unsafeStop() +{ + unsafeEvictPartitions(false, false); } size_t ExternalSortLocalPartitionWriter::unsafeEvictPartitions(bool, bool) @@ -367,6 +585,10 @@ size_t ExternalSortLocalPartitionWriter::unsafeEvictPartitions(bool, bool) IgnoreMemoryTracker ignore(settings.spill_memory_overhead); if (accumulated_blocks.empty()) return 0; + if (max_merge_block_bytes) + { + max_merge_block_size = std::max(max_merge_block_bytes / (current_accumulated_bytes / current_accumulated_rows), 128UL); + } Stopwatch watch; MergeSorter sorter(sort_header, std::move(accumulated_blocks), sort_description, max_merge_block_size, 0); streams.emplace_back(&tmp_data->createStream(sort_header)); @@ -378,6 +600,7 @@ size_t ExternalSortLocalPartitionWriter::unsafeEvictPartitions(bool, bool) streams.back()->finishWriting(); auto result = current_accumulated_bytes; current_accumulated_bytes = 0; + current_accumulated_rows = 0; shuffle_writer->split_result.total_spill_time += watch.elapsedNanoseconds(); return result; } @@ -562,8 +785,7 @@ size_t CelebornPartitionWriter::unsafeEvictSinglePartition(bool for_memory_spill { // escape memory track from current thread status; add untracked memory limit for create thread object, avoid trigger memory spill again IgnoreMemoryTracker ignore(settings.spill_memory_overhead); - ThreadFromGlobalPool thread(spill_to_celeborn); - thread.join(); + spill_to_celeborn(); } else { diff --git a/cpp-ch/local-engine/Shuffle/PartitionWriter.h b/cpp-ch/local-engine/Shuffle/PartitionWriter.h index 9c4e75db6efb..5b4285afda1f 100644 --- a/cpp-ch/local-engine/Shuffle/PartitionWriter.h +++ b/cpp-ch/local-engine/Shuffle/PartitionWriter.h @@ -17,7 +17,6 @@ #pragma once #include #include -#include #include #include #include @@ -26,6 +25,8 @@ #include #include +#include "CachedShuffleWriter.h" + namespace DB { class MergingSortedAlgorithm; @@ -33,17 +34,11 @@ class MergingSortedAlgorithm; namespace local_engine { -struct PartitionSpillInfo -{ - size_t partition_id; - size_t start; - size_t length; // in Bytes -}; struct SpillInfo { std::string spilled_file; - std::vector partition_spill_infos; + std::map> partition_spill_infos; }; class Partition @@ -113,7 +108,28 @@ class PartitionWriter : boost::noncopyable size_t last_partition_id; }; -class LocalPartitionWriter : public PartitionWriter +class Spillable +{ +public: + struct ExtraData + { + std::vector partition_block_buffer; + std::vector partition_buffer; + }; + + Spillable(SplitOptions options_) : split_options(std::move(options_)) {} + virtual ~Spillable() = default; + +protected: + String getNextSpillFile(); + std::vector mergeSpills(CachedShuffleWriter * shuffle_writer, WriteBuffer & data_file, ExtraData extra_data = {}); + std::vector spill_infos; + +private: + const SplitOptions split_options; +}; + +class LocalPartitionWriter : public PartitionWriter, public Spillable { public: explicit LocalPartitionWriter(CachedShuffleWriter * shuffle_writer); @@ -124,16 +140,79 @@ class LocalPartitionWriter : public PartitionWriter protected: size_t unsafeEvictPartitions(bool for_memory_spill, bool flush_block_buffer) override; void unsafeStop() override; +}; - String getNextSpillFile(); - std::vector mergeSpills(DB::WriteBuffer & data_file); +class SortBasedPartitionWriter : public PartitionWriter +{ +public: + explicit SortBasedPartitionWriter(CachedShuffleWriter * shuffle_writer_) : PartitionWriter(shuffle_writer_) + { + max_merge_block_size = options->split_size; + max_sort_buffer_size = options->max_sort_buffer_size; + max_merge_block_bytes = SerializedPlanParser::global_context->getSettings().prefer_external_sort_block_bytes; + } - std::vector spill_infos; + String getName() const override { return "SortBasedPartitionWriter"; } + void write(const PartitionInfo & info, DB::Block & block) override; + size_t adaptiveBlockSize() + { + size_t res = max_merge_block_size; + if (max_merge_block_bytes) + { + res = std::min(std::max(max_merge_block_bytes / (current_accumulated_bytes / current_accumulated_rows), 128UL), res); + } + return res; + } + +protected: + size_t max_merge_block_size = DB::DEFAULT_BLOCK_SIZE; + size_t max_sort_buffer_size = 1_GiB; + size_t max_merge_block_bytes = 0; + size_t current_accumulated_bytes = 0; + size_t current_accumulated_rows = 0; + Chunks accumulated_blocks; + Block output_header; + Block sort_header; + SortDescription sort_description; +}; + +class MemorySortLocalPartitionWriter : public SortBasedPartitionWriter, public Spillable +{ +public: + explicit MemorySortLocalPartitionWriter(CachedShuffleWriter* shuffle_writer_) + : SortBasedPartitionWriter(shuffle_writer_), Spillable(shuffle_writer_->options) + { + } + + ~MemorySortLocalPartitionWriter() override = default; + String getName() const override { return "MemorySortLocalPartitionWriter"; } + +protected: + size_t unsafeEvictPartitions(bool for_memory_spill, bool flush_block_buffer) override; + void unsafeStop() override; +}; + +class MemorySortCelebornPartitionWriter : public SortBasedPartitionWriter +{ +public: + explicit MemorySortCelebornPartitionWriter(CachedShuffleWriter* shuffle_writer_, std::unique_ptr celeborn_client_) + : SortBasedPartitionWriter(shuffle_writer_), celeborn_client(std::move(celeborn_client_)) + { + } + + ~MemorySortCelebornPartitionWriter() override = default; + +protected: + size_t unsafeEvictPartitions(bool for_memory_spill, bool flush_block_buffer) override; + void unsafeStop() override; + +private: + std::unique_ptr celeborn_client; }; class SortedPartitionDataMerger; -class ExternalSortLocalPartitionWriter : public PartitionWriter +class ExternalSortLocalPartitionWriter : public SortBasedPartitionWriter { public: struct MergeContext @@ -142,37 +221,30 @@ class ExternalSortLocalPartitionWriter : public PartitionWriter std::unique_ptr merger; }; - explicit ExternalSortLocalPartitionWriter(CachedShuffleWriter * shuffle_writer_) : PartitionWriter(shuffle_writer_) + explicit ExternalSortLocalPartitionWriter(CachedShuffleWriter * shuffle_writer_) : SortBasedPartitionWriter(shuffle_writer_) { max_merge_block_size = options->split_size; max_sort_buffer_size = options->max_sort_buffer_size; + max_merge_block_bytes = SerializedPlanParser::global_context->getSettings().prefer_external_sort_block_bytes; tmp_data = std::make_unique(SerializedPlanParser::global_context->getTempDataOnDisk()); } ~ExternalSortLocalPartitionWriter() override = default; String getName() const override { return "ExternalSortLocalPartitionWriter"; } - void write(const PartitionInfo & info, DB::Block & block) override; protected: size_t unsafeEvictPartitions(bool for_memory_spill, bool flush_block_buffer) override; /// Prepare for data merging, spill the remaining memory data，and create a merger object. MergeContext prepareMerge(); void unsafeStop() override; - std::queue mergeDataInMemory(); + std::queue mergeDataInMemory(); - size_t max_sort_buffer_size = 1_GiB; - size_t max_merge_block_size = DB::DEFAULT_BLOCK_SIZE; - size_t current_accumulated_bytes = 0; - DB::Chunks accumulated_blocks; - DB::Block output_header; - DB::Block sort_header; - DB::SortDescription sort_description; - DB::TemporaryDataOnDiskPtr tmp_data; - std::vector streams; + TemporaryDataOnDiskPtr tmp_data; + std::vector streams; }; -class ExternalSortCelebornPartitionWriter : public ExternalSortLocalPartitionWriter +class ExternalSortCelebornPartitionWriter : public ExternalSortLocalPartitionWriter { public: explicit ExternalSortCelebornPartitionWriter(CachedShuffleWriter * shuffle_writer_, std::unique_ptr celeborn_client_) diff --git a/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp b/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp index 5a5e969e1f58..7e3642dacd52 100644 --- a/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp +++ b/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp @@ -81,7 +81,7 @@ PartitionInfo RoundRobinSelectorBuilder::build(DB::Block & block) pid = pid_selection; pid_selection = (pid_selection + 1) % parts_num; } - return PartitionInfo::fromSelector(std::move(result), parts_num, use_external_sort_shuffle); + return PartitionInfo::fromSelector(std::move(result), parts_num, use_sort_shuffle); } HashSelectorBuilder::HashSelectorBuilder( @@ -156,7 +156,7 @@ PartitionInfo HashSelectorBuilder::build(DB::Block & block) } } } - return PartitionInfo::fromSelector(std::move(partition_ids), parts_num, use_external_sort_shuffle); + return PartitionInfo::fromSelector(std::move(partition_ids), parts_num, use_sort_shuffle); } @@ -177,7 +177,7 @@ PartitionInfo RangeSelectorBuilder::build(DB::Block & block) { DB::IColumn::Selector result; computePartitionIdByBinarySearch(block, result); - return PartitionInfo::fromSelector(std::move(result), partition_num, use_external_sort_shuffle); + return PartitionInfo::fromSelector(std::move(result), partition_num, use_sort_shuffle); } void RangeSelectorBuilder::initSortInformation(Poco::JSON::Array::Ptr orderings) diff --git a/cpp-ch/local-engine/Shuffle/SelectorBuilder.h b/cpp-ch/local-engine/Shuffle/SelectorBuilder.h index 9510291c8864..97894daa3c14 100644 --- a/cpp-ch/local-engine/Shuffle/SelectorBuilder.h +++ b/cpp-ch/local-engine/Shuffle/SelectorBuilder.h @@ -46,11 +46,12 @@ struct PartitionInfo class SelectorBuilder { public: - explicit SelectorBuilder(bool use_external_sort_shuffle) : use_external_sort_shuffle(use_external_sort_shuffle) { } + explicit SelectorBuilder(bool use_external_sort_shuffle) : use_sort_shuffle(use_external_sort_shuffle) { } virtual ~SelectorBuilder() = default; virtual PartitionInfo build(DB::Block & block) = 0; + void setUseSortShuffle(bool use_external_sort_shuffle_) { use_sort_shuffle = use_external_sort_shuffle_; } protected: - bool use_external_sort_shuffle = false; + bool use_sort_shuffle = false; }; class RoundRobinSelectorBuilder : public SelectorBuilder diff --git a/cpp-ch/local-engine/Shuffle/ShuffleSplitter.h b/cpp-ch/local-engine/Shuffle/ShuffleSplitter.h index cfd4062611ed..75edea325c67 100644 --- a/cpp-ch/local-engine/Shuffle/ShuffleSplitter.h +++ b/cpp-ch/local-engine/Shuffle/ShuffleSplitter.h @@ -53,7 +53,8 @@ struct SplitOptions size_t max_sort_buffer_size = 1_GiB; // Whether to spill firstly before stop external sort shuffle. bool spill_firstly_before_stop = true; - bool force_sort = true; + bool force_external_sort = false; + bool force_mermory_sort = false; }; class ColumnsBuffer diff --git a/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.h b/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.h index 31f5547fba55..e38f58647e96 100644 --- a/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.h +++ b/cpp-ch/local-engine/Shuffle/SortedPartitionDataMerger.h @@ -22,6 +22,8 @@ namespace local_engine { + +int64_t searchLastPartitionIdIndex(DB::ColumnPtr column, size_t start, size_t partition_id); class SortedPartitionDataMerger; using SortedPartitionDataMergerPtr = std::unique_ptr; class SortedPartitionDataMerger diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index c7721b470cf8..00a7b1b0ad82 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -671,7 +671,8 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na jboolean flush_block_buffer_before_evict, jlong max_sort_buffer_size, jboolean spill_firstly_before_stop, - jboolean force_sort) + jboolean force_external_sort, + jboolean force_memory_sort) { LOCAL_ENGINE_JNI_METHOD_START std::string hash_exprs; @@ -718,7 +719,8 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na .flush_block_buffer_before_evict = static_cast(flush_block_buffer_before_evict), .max_sort_buffer_size = static_cast(max_sort_buffer_size), .spill_firstly_before_stop = static_cast(spill_firstly_before_stop), - .force_sort = static_cast(force_sort) + .force_external_sort = static_cast(force_external_sort), + .force_mermory_sort = static_cast(force_memory_sort) }; auto name = jstring2string(env, short_name); local_engine::SplitterHolder * splitter; @@ -745,7 +747,9 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na jstring hash_algorithm, jobject pusher, jboolean throw_if_memory_exceed, - jboolean flush_block_buffer_before_evict) + jboolean flush_block_buffer_before_evict, + jboolean force_external_sort, + jboolean force_memory_sort) { LOCAL_ENGINE_JNI_METHOD_START std::string hash_exprs; @@ -780,7 +784,10 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na .spill_threshold = static_cast(spill_threshold), .hash_algorithm = jstring2string(env, hash_algorithm), .throw_if_memory_exceed = static_cast(throw_if_memory_exceed), - .flush_block_buffer_before_evict = static_cast(flush_block_buffer_before_evict)}; + .flush_block_buffer_before_evict = static_cast(flush_block_buffer_before_evict), + .force_external_sort = static_cast(force_external_sort), + .force_mermory_sort = static_cast(force_memory_sort) + }; auto name = jstring2string(env, short_name); local_engine::SplitterHolder * splitter; splitter = new local_engine::SplitterHolder{.splitter = std::make_unique(name, options, pusher)}; diff --git a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala index 75efa355302d..524a3ee2e464 100644 --- a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala +++ b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala @@ -78,7 +78,9 @@ class CHCelebornHashBasedColumnarShuffleWriter[K, V]( CHBackendSettings.shuffleHashAlgorithm, celebornPartitionPusher, GlutenConfig.getConf.chColumnarThrowIfMemoryExceed, - GlutenConfig.getConf.chColumnarFlushBlockBufferBeforeEvict + GlutenConfig.getConf.chColumnarFlushBlockBufferBeforeEvict, + GlutenConfig.getConf.chColumnarForceExternalSortShuffle, + GlutenConfig.getConf.chColumnarForceMemorySortShuffle ) CHNativeMemoryAllocators.createSpillable( "CelebornShuffleWriter", diff --git a/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarSortShuffleAQESuite.scala b/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarExternalSortShuffleSuite.scala similarity index 97% rename from gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarSortShuffleAQESuite.scala rename to gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarExternalSortShuffleSuite.scala index 0072fe8c95d6..3ecf1fc1ac7b 100644 --- a/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarSortShuffleAQESuite.scala +++ b/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarExternalSortShuffleSuite.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.execution.CoalescedPartitionSpec import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec} import org.apache.spark.sql.internal.SQLConf -class GlutenClickHouseRSSColumnarSortShuffleAQESuite +class GlutenClickHouseRSSColumnarExternalSortShuffleSuite extends GlutenClickHouseTPCHAbstractSuite with AdaptiveSparkPlanHelper { @@ -45,7 +45,7 @@ class GlutenClickHouseRSSColumnarSortShuffleAQESuite .set("spark.sql.adaptive.enabled", "true") .set("spark.shuffle.service.enabled", "false") .set("spark.celeborn.client.spark.shuffle.writer", "hash") - .set("spark.gluten.sql.columnar.backend.ch.forceSortShuffle", "true") + .set("spark.gluten.sql.columnar.backend.ch.forceExternalSortShuffle", "true") } test("TPCH Q1") { diff --git a/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarMemorySortShuffleSuite.scala b/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarMemorySortShuffleSuite.scala new file mode 100644 index 000000000000..ddef1d87cd08 --- /dev/null +++ b/gluten-celeborn/clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseRSSColumnarMemorySortShuffleSuite.scala @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.SparkConf +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + +class GlutenClickHouseRSSColumnarMemorySortShuffleSuite + extends GlutenClickHouseTPCHAbstractSuite + with AdaptiveSparkPlanHelper { + + override protected val tablesPath: String = basePath + "/tpch-data-ch" + override protected val tpchQueries: String = rootPath + "queries/tpch-queries-ch" + override protected val queriesResults: String = + rootPath + "../../../../../backends-clickhouse/src/test/resources/mergetree-queries-output" + + override protected val parquetTableDataPath: String = + "../../../../../gluten-core/src/test/resources/tpch-data" + + /** Run Gluten + ClickHouse Backend with ColumnarShuffleManager */ + override protected def sparkConf: SparkConf = { + super.sparkConf + .set( + "spark.shuffle.manager", + "org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager") + .set("spark.io.compression.codec", "LZ4") + .set("spark.sql.shuffle.partitions", "5") + .set("spark.sql.autoBroadcastJoinThreshold", "10MB") + .set("spark.sql.adaptive.enabled", "true") + .set("spark.shuffle.service.enabled", "false") + .set("spark.celeborn.client.spark.shuffle.writer", "hash") + .set("spark.gluten.sql.columnar.backend.ch.forceMemorySortShuffle", "true") + } + + test("TPCH Q1") { + runTPCHQuery(1) { df => } + } + + test("TPCH Q2") { + runTPCHQuery(2) { df => } + } + + test("TPCH Q3") { + runTPCHQuery(3) { df => } + } + + test("TPCH Q4") { + runTPCHQuery(4) { df => } + } + + test("TPCH Q5") { + runTPCHQuery(5) { df => } + } + + test("TPCH Q6") { + runTPCHQuery(6) { df => } + } + + test("TPCH Q7") { + runTPCHQuery(7) { df => } + } + + test("TPCH Q8") { + runTPCHQuery(8) { df => } + } + + test("TPCH Q9") { + runTPCHQuery(9) { df => } + } + + test("TPCH Q10") { + runTPCHQuery(10) { df => } + } + + test("TPCH Q11") { + runTPCHQuery(11) { df => } + } + + test("TPCH Q12") { + runTPCHQuery(12) { df => } + } + + test("TPCH Q13") { + runTPCHQuery(13) { df => } + } + + test("TPCH Q14") { + runTPCHQuery(14) { df => } + } + + test("TPCH Q15") { + runTPCHQuery(15) { df => } + } + + test("TPCH Q16") { + runTPCHQuery(16, noFallBack = false) { df => } + } + + test("TPCH Q17") { + runTPCHQuery(17) { df => } + } + + test("TPCH Q18") { + runTPCHQuery(18) { df => } + } + + test("TPCH Q19") { + runTPCHQuery(19) { df => } + } + + test("TPCH Q20") { + runTPCHQuery(20) { df => } + } + + test("TPCH Q21") { + runTPCHQuery(21, noFallBack = false) { df => } + } + + test("TPCH Q22") { + runTPCHQuery(22) { df => } + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index c9a62b8b748d..6659a42c71f9 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -297,7 +297,14 @@ class GlutenConfig(conf: SQLConf) extends Logging { def chColumnarShufflePreferSpill: Boolean = conf.getConf(COLUMNAR_CH_SHUFFLE_PREFER_SPILL_ENABLED) - def chColumnarShuffleSpillThreshold: Long = conf.getConf(COLUMNAR_CH_SHUFFLE_SPILL_THRESHOLD) + def chColumnarShuffleSpillThreshold: Long = { + val threshold = conf.getConf(COLUMNAR_CH_SHUFFLE_SPILL_THRESHOLD) + if (threshold == 0) { + (conf.getConf(COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES) * 0.9).toLong + } else { + threshold + } + } def chColumnarThrowIfMemoryExceed: Boolean = conf.getConf(COLUMNAR_CH_THROW_IF_MEMORY_EXCEED) @@ -309,7 +316,11 @@ class GlutenConfig(conf: SQLConf) extends Logging { def chColumnarSpillFirstlyBeforeStop: Boolean = conf.getConf(COLUMNAR_CH_SPILL_FIRSTLY_BEFORE_STOP) - def chColumnarForceSortShuffle: Boolean = conf.getConf(COLUMNAR_CH_FORCE_SORT_SHUFFLE) + def chColumnarForceExternalSortShuffle: Boolean = + conf.getConf(COLUMNAR_CH_FORCE_EXTERNAL_SORT_SHUFFLE) + + def chColumnarForceMemorySortShuffle: Boolean = + conf.getConf(COLUMNAR_CH_FORCE_MEMORY_SORT_SHUFFLE) def cartesianProductTransformerEnabled: Boolean = conf.getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED) @@ -1416,7 +1427,7 @@ object GlutenConfig { .internal() .doc("The maximum size of sort shuffle buffer in CH backend.") .bytesConf(ByteUnit.BYTE) - .createWithDefaultString("1GB") + .createWithDefaultString("0") val COLUMNAR_CH_SPILL_FIRSTLY_BEFORE_STOP = buildConf("spark.gluten.sql.columnar.backend.ch.spillFirstlyBeforeStop") @@ -1425,11 +1436,17 @@ object GlutenConfig { .booleanConf .createWithDefault(true) - val COLUMNAR_CH_FORCE_SORT_SHUFFLE = - buildConf("spark.gluten.sql.columnar.backend.ch.forceSortShuffle") + val COLUMNAR_CH_FORCE_EXTERNAL_SORT_SHUFFLE = + buildConf("spark.gluten.sql.columnar.backend.ch.forceExternalSortShuffle") + .internal() + .doc("Whether to force to use external sort shuffle in CH backend. ") + .booleanConf + .createWithDefault(false) + + val COLUMNAR_CH_FORCE_MEMORY_SORT_SHUFFLE = + buildConf("spark.gluten.sql.columnar.backend.ch.forceMemorySortShuffle") .internal() - .doc("Whether to force to use sort shuffle in CH backend. " + - "Sort shuffle will enable When partition num greater than 300.") + .doc("Whether to force to use memory sort shuffle in CH backend. ") .booleanConf .createWithDefault(false) From e5dcbe3884d5215cc652246476b1ec980c859d4c Mon Sep 17 00:00:00 2001 From: Jacky Lee Date: Thu, 30 May 2024 16:08:45 +0800 Subject: [PATCH 176/402] [GLUTEN-5656][CORE][FOLLOWUP] Support GetStructField with NullLiteralNode as subqueries not executing during validation (#5923) --- .../org/apache/gluten/expression/ExpressionTransformer.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala index 0f0eb2969f7e..51b19ab140d9 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/ExpressionTransformer.scala @@ -19,6 +19,7 @@ package org.apache.gluten.expression import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression.ConverterUtils.FunctionConfig import org.apache.gluten.expression.ExpressionConverter.replaceWithExpressionTransformer +import org.apache.gluten.substrait.`type`.StructNode import org.apache.gluten.substrait.expression._ import org.apache.spark.sql.catalyst.expressions._ @@ -63,6 +64,10 @@ case class VeloxGetStructFieldTransformer( case node: SelectionNode => // Append the nested index to selection node. node.addNestedChildIdx(JInteger.valueOf(original.ordinal)) + case node: NullLiteralNode => + val nodeType = + node.getTypeNode.asInstanceOf[StructNode].getFieldTypes.get(original.ordinal) + ExpressionBuilder.makeNullLiteral(nodeType) case other => throw new GlutenNotSupportException(s"$other is not supported.") } From c620f4f3fa215906125f6c3e979c76dea8a5b30c Mon Sep 17 00:00:00 2001 From: Shuai li Date: Thu, 30 May 2024 17:04:17 +0800 Subject: [PATCH 177/402] [GLUTEN-5898][CH] Fix regexp_extract function use bracket has diff behaver with spark (#5908) [CH] Fix regexp_extract function use bracket has diff behaver with spark --- .../GlutenFunctionValidateSuite.scala | 20 +++++ .../scalar_function_parser/regexp_extract.cpp | 80 ++++++++++++++++++- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index 7b52a970ef08..5a1ca679986f 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -708,4 +708,24 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS } + test("GLUTEN-5897: fix regexp_extract with bracket") { + withTable("regexp_extract_bracket") { + sql("create table regexp_extract_bracket(a String) using parquet") + sql( + """ + |insert into regexp_extract_bracket values ('123.123abc-abc'),('123-LOW'),('123]abc-abc') + |""".stripMargin) + + val sql_str = + s"""select + | regexp_extract(a, '([0-9][[\\.][0-9]]*)', 1) + | , regexp_extract(a, '([0-9][[\\.][0-9]]*)', 1) + | , regexp_extract(a, '([0-9][[]]]*)', 1) + | from regexp_extract_bracket + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + } diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/regexp_extract.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/regexp_extract.cpp index 2c0eeff1ccd3..8f75baf689b2 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/regexp_extract.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/regexp_extract.cpp @@ -15,6 +15,8 @@ * limitations under the License. */ +#include + #include namespace DB @@ -56,10 +58,11 @@ class FunctionParserRegexpExtract : public FunctionParser size_t expr_size = expr_str.size(); if (expr_str.data()[expr_size - 1] == '$') expr_str.replace(expr_str.find_last_of("$"), 1, "(?:(\n)*)$"); - - const auto * regex_expr_node = addColumnToActionsDAG(actions_dag, std::make_shared(), expr_str); + + String sparkRegexp = adjustSparkRegexpRule(expr_str); + const auto * regex_expr_node = addColumnToActionsDAG(actions_dag, std::make_shared(), sparkRegexp); auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); - parsed_args[1] = regex_expr_node; + parsed_args[1] = regex_expr_node; const auto * result_node = toFunctionNode(actions_dag, "regexpExtract", parsed_args); return convertNodeTypeIfNeeded(substrait_func, result_node, actions_dag); } @@ -69,6 +72,77 @@ class FunctionParserRegexpExtract : public FunctionParser else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} 2nd argument's type must be const", getName()); } + +private: + String adjustSparkRegexpRule(String & str) const + { + const auto left_bracket_pos = str.find('['); + const auto right_bracket_pos = str.find(']'); + + if (left_bracket_pos == str.npos || right_bracket_pos == str.npos || left_bracket_pos >= right_bracket_pos) + return str; + + auto throw_message = [this, &str]() -> void { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value of parameter(s) 'regexp' in `{}` is invalid: '{}'", getName(), str); + }; + + ReadBufferFromString buf(str); + std::stack strs; + strs.emplace(""); + bool nead_right_bracket = false; + + while (!buf.eof()) + { + if (*buf.position() == '[') + { + strs.emplace(""); + } + else if (*buf.position() == ']') + { + if (strs.size() == 1) + { + // "ab]c" + strs.top().append("]"); + } + else + { + String back = strs.top(); + strs.pop(); + if (strs.size() == 1) + { + // "abc[abc]abc" + strs.top().append("[").append(back).append("]"); + nead_right_bracket = false; + } + else + { + // "abc[a[abc]c]abc" + strs.top().append(back); + nead_right_bracket = true; + } + } + } + else + { + strs.top() += *buf.position(); + } + + ++buf.position(); + } + + if (nead_right_bracket && strs.size() != 1) + throw_message(); + + while (strs.size() != 1) + { + String back = strs.top(); + strs.pop(); + strs.top().append("[").append(back); + } + + return strs.top(); + } }; static FunctionParserRegister register_regexp_extract; From 6a28f214f912b00bd4b42f3f587760bdd0a93a46 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Thu, 30 May 2024 17:30:34 +0800 Subject: [PATCH 178/402] [GLUTEN-5691][CH] Enable merge on local disk first after insert into mergetree (#5692) [CH] Enable merge on local disk first after insert into mergetree --- ...tenClickHouseMergeTreeWriteOnS3Suite.scala | 26 ++ ...ClickHouseWholeStageTransformerSuite.scala | 4 +- cpp-ch/local-engine/Common/CHUtil.cpp | 28 +- cpp-ch/local-engine/Common/MergeTreeTool.cpp | 6 +- .../Parser/MergeTreeRelParser.cpp | 103 +++--- .../local-engine/Parser/MergeTreeRelParser.h | 13 +- .../Storages/CustomStorageMergeTree.cpp | 1 + .../Mergetree/SparkMergeTreeWriter.cpp | 310 +++++++++++------- .../Storages/Mergetree/SparkMergeTreeWriter.h | 24 +- .../Storages/StorageMergeTreeFactory.cpp | 53 ++- .../Storages/StorageMergeTreeFactory.h | 8 +- cpp-ch/local-engine/local_engine_jni.cpp | 26 +- .../GlutenWriterColumnarRules.scala | 7 +- 13 files changed, 355 insertions(+), 254 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala index 2b641438802f..44c2af76f933 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala @@ -633,5 +633,31 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite .count() assert(result == 600572) } + + test("test mergetree insert with optimize basic") { + val table_name = "lineitem_mergetree_insert_optimize_basic_s3" + val dataPath = s"s3a://$BUCKET_NAME/$table_name" + + withSQLConf( + ("spark.databricks.delta.optimize.minFileSize" -> "200000000"), + ("spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true") + ) { + spark.sql(s""" + |DROP TABLE IF EXISTS $table_name; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS $table_name + |USING clickhouse + |LOCATION '$dataPath' + | as select * from lineitem + |""".stripMargin) + + val ret = spark.sql(s"select count(*) from $table_name").collect() + assert(ret.apply(0).get(0) == 600572) + assert( + !new File(s"$CH_DEFAULT_STORAGE_DIR/lineitem_mergetree_insert_optimize_basic").exists()) + } + } } // scalastyle:off line.size.limit diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala index a891d6d1027b..9412326ae342 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala @@ -55,6 +55,8 @@ class GlutenClickHouseWholeStageTransformerSuite extends WholeStageTransformerSu val S3_ACCESS_KEY = "BypTYzcXOlfr03FFIvt4" val S3_SECRET_KEY = "K9MDaGItPSaphorZM8t4hXf30gHF9dBWi6L2dK5E" + val CH_DEFAULT_STORAGE_DIR = "/data" + def AlmostEqualsIsRel(expected: Double, actual: Double, EPSILON: Double = DBL_EPSILON): Unit = { val diff = Math.abs(expected - actual) val epsilon = EPSILON * Math.max(Math.abs(expected), Math.abs(actual)) @@ -162,7 +164,7 @@ class GlutenClickHouseWholeStageTransformerSuite extends WholeStageTransformerSu override def beforeAll(): Unit = { // is not exist may cause some ut error - assert(new File("/data").exists()) + assert(new File(CH_DEFAULT_STORAGE_DIR).exists()) // prepare working paths val basePathDir = new File(basePath) diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 317adda2ae3e..42787172af53 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -552,7 +552,11 @@ DB::Context::ConfigurationPtr BackendInitializerUtil::initConfig(std::mapsetString(key.substr(CH_RUNTIME_CONFIG_PREFIX.size()), value); + const auto name = key.substr(CH_RUNTIME_CONFIG_PREFIX.size()); + if ((name == "storage_configuration.disks.s3.metadata_path" || name == "path") && !value.ends_with("/")) + config->setString(name, value + "/"); + else + config->setString(name, value); } } @@ -722,6 +726,11 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) global_context->setTemporaryStoragePath(config->getString("tmp_path", getDefaultPath()), 0); global_context->setPath(config->getString("path", "/")); + String uncompressed_cache_policy = config->getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY); + size_t uncompressed_cache_size = config->getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE); + double uncompressed_cache_size_ratio = config->getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO); + global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio); + String mark_cache_policy = config->getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY); size_t mark_cache_size = config->getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE); double mark_cache_size_ratio = config->getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO); @@ -730,10 +739,21 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); + String index_uncompressed_cache_policy = config->getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY); + size_t index_uncompressed_cache_size = config->getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); + double index_uncompressed_cache_size_ratio = config->getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); + global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio); + String index_mark_cache_policy = config->getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); size_t index_mark_cache_size = config->getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); double index_mark_cache_size_ratio = config->getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio); + + size_t mmap_cache_size = config->getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE); + global_context->setMMappedFileCache(mmap_cache_size); + + /// Initialize a dummy query cache. + global_context->setQueryCache(0, 0, 0, 0); } } @@ -861,6 +881,12 @@ void BackendInitializerUtil::init(std::string * plan) active_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded active_parts_loading_threads); + + const size_t cleanup_threads = config->getUInt("max_parts_cleaning_thread_pool_size", 128); + getPartsCleaningThreadPool().initialize( + cleanup_threads, + 0, // We don't need any threads one all the parts will be deleted + cleanup_threads); }); } diff --git a/cpp-ch/local-engine/Common/MergeTreeTool.cpp b/cpp-ch/local-engine/Common/MergeTreeTool.cpp index f430c9306912..63bf64726bf2 100644 --- a/cpp-ch/local-engine/Common/MergeTreeTool.cpp +++ b/cpp-ch/local-engine/Common/MergeTreeTool.cpp @@ -16,14 +16,14 @@ */ #include "MergeTreeTool.h" +#include +#include + #include #include #include #include #include -#include -#include -#include #include using namespace DB; diff --git a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp index c89632919e29..9afa83973cd8 100644 --- a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp +++ b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp @@ -66,21 +66,18 @@ MergeTreeTable MergeTreeRelParser::parseMergeTreeTable(const substrait::ReadRel: return parseMergeTreeTableString(table.value()); } -CustomStorageMergeTreePtr MergeTreeRelParser::parseStorage(const MergeTreeTable & merge_tree_table, ContextMutablePtr context, UUID uuid) +CustomStorageMergeTreePtr +MergeTreeRelParser::parseStorage(const MergeTreeTable & merge_tree_table, ContextMutablePtr context, bool restore) { DB::Block header = TypeParser::buildBlockFromNamedStruct(merge_tree_table.schema, merge_tree_table.low_card_key); auto names_and_types_list = header.getNamesAndTypesList(); - auto storage_factory = StorageMergeTreeFactory::instance(); auto metadata = buildMetaData(names_and_types_list, context, merge_tree_table); - { - // use instance global table (without uuid) to restore metadata folder on current instance - // we need its lock - - auto global_storage = storage_factory.getStorage( + // use instance global table (without uuid) to restore metadata folder on current instance + // we need its lock + auto global_storage = StorageMergeTreeFactory::getStorage( StorageID(merge_tree_table.database, merge_tree_table.table), merge_tree_table.snapshot_id, - metadata->getColumns(), [&]() -> CustomStorageMergeTreePtr { auto custom_storage_merge_tree = std::make_shared( @@ -95,42 +92,47 @@ CustomStorageMergeTreePtr MergeTreeRelParser::parseStorage(const MergeTreeTable return custom_storage_merge_tree; }); + if (restore) restoreMetaData(global_storage, merge_tree_table, *context); - } - // return local table (with a uuid) for isolation - auto storage = storage_factory.getStorage( - StorageID(merge_tree_table.database, merge_tree_table.table, uuid), - merge_tree_table.snapshot_id, - metadata->getColumns(), - [&]() -> CustomStorageMergeTreePtr - { - auto custom_storage_merge_tree = std::make_shared( - StorageID(merge_tree_table.database, merge_tree_table.table, uuid), - merge_tree_table.relative_path, - *metadata, - false, - context, - "", - MergeTreeData::MergingParams(), - buildMergeTreeSettings(merge_tree_table.table_configs)); - return custom_storage_merge_tree; - }); - return storage; + return global_storage; } CustomStorageMergeTreePtr -MergeTreeRelParser::parseStorage(const substrait::ReadRel::ExtensionTable & extension_table, ContextMutablePtr context, UUID uuid) +MergeTreeRelParser::parseStorage(const substrait::ReadRel::ExtensionTable & extension_table, ContextMutablePtr context) { auto merge_tree_table = parseMergeTreeTable(extension_table); - return parseStorage(merge_tree_table, context, uuid); + return parseStorage(merge_tree_table, context, true); +} + +CustomStorageMergeTreePtr +MergeTreeRelParser::copyToDefaultPolicyStorage(MergeTreeTable merge_tree_table, ContextMutablePtr context) +{ + auto temp_uuid = UUIDHelpers::generateV4(); + String temp_uuid_str = toString(temp_uuid); + merge_tree_table.table = merge_tree_table.table + "_" + temp_uuid_str; + merge_tree_table.snapshot_id = ""; + merge_tree_table.table_configs.storage_policy = ""; + merge_tree_table.relative_path = merge_tree_table.relative_path + "_" + temp_uuid_str; + return parseStorage(merge_tree_table, context); +} + +CustomStorageMergeTreePtr +MergeTreeRelParser::copyToVirtualStorage(MergeTreeTable merge_tree_table, ContextMutablePtr context) +{ + auto temp_uuid = UUIDHelpers::generateV4(); + String temp_uuid_str = toString(temp_uuid); + merge_tree_table.table = merge_tree_table.table + "_" + temp_uuid_str; + merge_tree_table.snapshot_id = ""; + return parseStorage(merge_tree_table, context); } DB::QueryPlanPtr MergeTreeRelParser::parseReadRel( DB::QueryPlanPtr query_plan, const substrait::ReadRel & rel, const substrait::ReadRel::ExtensionTable & extension_table) { auto merge_tree_table = parseMergeTreeTable(extension_table); - DB::Block header = TypeParser::buildBlockFromNamedStruct(merge_tree_table.schema, merge_tree_table.low_card_key); + auto storage = parseStorage(extension_table, global_context); + DB::Block input; if (rel.has_base_schema() && rel.base_schema().names_size()) { @@ -139,35 +141,15 @@ DB::QueryPlanPtr MergeTreeRelParser::parseReadRel( else { NamesAndTypesList one_column_name_type; - one_column_name_type.push_back(header.getNamesAndTypesList().front()); + one_column_name_type.push_back(storage->getInMemoryMetadataPtr()->getColumns().getAll().front()); input = BlockUtil::buildHeader(one_column_name_type); - LOG_DEBUG(&Poco::Logger::get("SerializedPlanParser"), "Try to read ({}) instead of empty header", header.dumpNames()); + LOG_DEBUG( + &Poco::Logger::get("SerializedPlanParser"), "Try to read ({}) instead of empty header", one_column_name_type.front().dump()); } - auto storage_factory = StorageMergeTreeFactory::instance(); - auto metadata = buildMetaData(header.getNamesAndTypesList(), context, merge_tree_table); - StorageID table_id(merge_tree_table.database, merge_tree_table.table); - auto storage = storage_factory.getStorage( - table_id, - merge_tree_table.snapshot_id, - metadata->getColumns(), - [&]() -> CustomStorageMergeTreePtr - { - auto custom_storage_merge_tree = std::make_shared( - StorageID(merge_tree_table.database, merge_tree_table.table), - merge_tree_table.relative_path, - *metadata, - false, - global_context, - "", - MergeTreeData::MergingParams(), - buildMergeTreeSettings(merge_tree_table.table_configs)); - return custom_storage_merge_tree; - }); - restoreMetaData(storage, merge_tree_table, *context); for (const auto & [name, sizes] : storage->getColumnSizes()) column_sizes[name] = sizes.data_compressed; - auto storage_snapshot = std::make_shared(*storage, metadata); + auto storage_snapshot = std::make_shared(*storage, storage->getInMemoryMetadataPtr()); auto names_and_types_list = input.getNamesAndTypesList(); auto query_info = buildQueryInfo(names_and_types_list); @@ -180,9 +162,9 @@ DB::QueryPlanPtr MergeTreeRelParser::parseReadRel( query_info->prewhere_info = parsePreWhereInfo(rel.filter(), input); } - std::vector selected_parts = storage_factory.getDataParts(table_id, merge_tree_table.snapshot_id, merge_tree_table.getPartNames()); - if (selected_parts.empty()) - throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "no data part found."); + std::vector selected_parts + = StorageMergeTreeFactory::getDataPartsByNames(storage->getStorageID(), merge_tree_table.snapshot_id, merge_tree_table.getPartNames()); + auto read_step = storage->reader.readFromParts( selected_parts, /* alter_conversions = */ @@ -195,8 +177,7 @@ DB::QueryPlanPtr MergeTreeRelParser::parseReadRel( 1); auto * source_step_with_filter = static_cast(read_step.get()); - const auto & storage_prewhere_info = query_info->prewhere_info; - if (storage_prewhere_info) + if (const auto & storage_prewhere_info = query_info->prewhere_info) { source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions, storage_prewhere_info->prewhere_column_name); source_step_with_filter->applyFilters(); @@ -427,7 +408,7 @@ String MergeTreeRelParser::filterRangesOnDriver(const substrait::ReadRel & read_ auto storage_factory = StorageMergeTreeFactory::instance(); std::vector selected_parts - = storage_factory.getDataParts(StorageID(merge_tree_table.database, merge_tree_table.table), merge_tree_table.snapshot_id, merge_tree_table.getPartNames()); + = storage_factory.getDataPartsByNames(StorageID(merge_tree_table.database, merge_tree_table.table), merge_tree_table.snapshot_id, merge_tree_table.getPartNames()); auto storage_snapshot = std::make_shared(*custom_storage_mergetree, custom_storage_mergetree->getInMemoryMetadataPtr()); if (selected_parts.empty()) diff --git a/cpp-ch/local-engine/Parser/MergeTreeRelParser.h b/cpp-ch/local-engine/Parser/MergeTreeRelParser.h index 7619851d93b6..bf27b184f987 100644 --- a/cpp-ch/local-engine/Parser/MergeTreeRelParser.h +++ b/cpp-ch/local-engine/Parser/MergeTreeRelParser.h @@ -41,9 +41,18 @@ class MergeTreeRelParser : public RelParser { public: static CustomStorageMergeTreePtr - parseStorage(const substrait::ReadRel::ExtensionTable & extension_table, ContextMutablePtr context, UUID uuid = UUIDHelpers::Nil); + parseStorage(const substrait::ReadRel::ExtensionTable & extension_table, ContextMutablePtr context); + static CustomStorageMergeTreePtr parseStorage( + const MergeTreeTable & merge_tree_table, ContextMutablePtr context, bool restore = false); + + // Create random table name and table path and use default storage policy. + // In insert case, mergetree data can be upload after merges in default storage(Local Disk). + static CustomStorageMergeTreePtr + copyToDefaultPolicyStorage(MergeTreeTable merge_tree_table, ContextMutablePtr context); + + // Use same table path and data path as the originial table. static CustomStorageMergeTreePtr - parseStorage(const MergeTreeTable & merge_tree_table, ContextMutablePtr context, UUID uuid = UUIDHelpers::Nil); + copyToVirtualStorage(MergeTreeTable merge_tree_table, ContextMutablePtr context); static MergeTreeTable parseMergeTreeTable(const substrait::ReadRel::ExtensionTable & extension_table); diff --git a/cpp-ch/local-engine/Storages/CustomStorageMergeTree.cpp b/cpp-ch/local-engine/Storages/CustomStorageMergeTree.cpp index 368015fb9278..ec0e0932fc76 100644 --- a/cpp-ch/local-engine/Storages/CustomStorageMergeTree.cpp +++ b/cpp-ch/local-engine/Storages/CustomStorageMergeTree.cpp @@ -150,6 +150,7 @@ std::atomic CustomStorageMergeTree::part_num; std::vector CustomStorageMergeTree::loadDataPartsWithNames(std::unordered_set parts) { + auto parts_lock = lockParts(); std::vector data_parts; const auto disk = getStoragePolicy()->getDisks().at(0); for (const auto& name : parts) diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp index cdb7d3455680..259af5698aa1 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp @@ -16,12 +16,15 @@ */ #include "SparkMergeTreeWriter.h" +#include + #include #include #include +#include #include #include -#include +#include #include @@ -40,7 +43,7 @@ using namespace DB; namespace local_engine { -Block removeColumnSuffix(const DB::Block & block) +Block removeColumnSuffix(const Block & block) { ColumnsWithTypeAndName columns; for (int i = 0; i < block.columns(); ++i) @@ -55,54 +58,76 @@ Block removeColumnSuffix(const DB::Block & block) } SparkMergeTreeWriter::SparkMergeTreeWriter( - CustomStorageMergeTreePtr storage_, - const DB::StorageMetadataPtr & metadata_snapshot_, + const MergeTreeTable & merge_tree_table, const DB::ContextPtr & context_, - const String & uuid_, + const String & part_name_prefix_, const String & partition_dir_, const String & bucket_dir_) - : storage(storage_) - , metadata_snapshot(metadata_snapshot_) - , context(context_) - , uuid(uuid_) + : context(context_) + , part_name_prefix(part_name_prefix_) , partition_dir(partition_dir_) , bucket_dir(bucket_dir_) , thread_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1, 1, 100000) { + dest_storage = MergeTreeRelParser::parseStorage(merge_tree_table, SerializedPlanParser::global_context); + + if (dest_storage->getStoragePolicy()->getAnyDisk()->isRemote()) + { + isRemoteStorage = true; + temp_storage = MergeTreeRelParser::copyToDefaultPolicyStorage(merge_tree_table, SerializedPlanParser::global_context); + storage = temp_storage; + LOG_DEBUG( + &Poco::Logger::get("SparkMergeTreeWriter"), + "Create temp table {} for local merge.", + temp_storage->getStorageID().getFullNameNotQuoted()); + } + else + storage = dest_storage; + + metadata_snapshot = storage->getInMemoryMetadataPtr(); + header = metadata_snapshot->getSampleBlock(); const DB::Settings & settings = context->getSettingsRef(); squashing_transform = std::make_unique(settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes); if (!partition_dir.empty()) - { extractPartitionValues(partition_dir, partition_values); - } - header = metadata_snapshot->getSampleBlock(); Field is_merge; - if (context->getSettings().tryGet("mergetree.merge_after_insert", is_merge)) + if (settings.tryGet("mergetree.merge_after_insert", is_merge)) merge_after_insert = is_merge.get(); Field limit_size_field; - if (context->getSettings().tryGet("optimize.minFileSize", limit_size_field)) + if (settings.tryGet("optimize.minFileSize", limit_size_field)) merge_min_size = limit_size_field.get() <= 0 ? merge_min_size : limit_size_field.get(); Field limit_cnt_field; - if (context->getSettings().tryGet("mergetree.max_num_part_per_merge_task", limit_cnt_field)) + if (settings.tryGet("mergetree.max_num_part_per_merge_task", limit_cnt_field)) merge_limit_parts = limit_cnt_field.get() <= 0 ? merge_limit_parts : limit_cnt_field.get(); } void SparkMergeTreeWriter::write(DB::Block & block) { auto new_block = removeColumnSuffix(block); - auto converter = ActionsDAG::makeConvertingActions(new_block.getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), DB::ActionsDAG::MatchColumnsMode::Position);; - if (converter) + if (auto converter = ActionsDAG::makeConvertingActions( + new_block.getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), DB::ActionsDAG::MatchColumnsMode::Position)) + ExpressionActions(converter).execute(new_block); + + if (auto add_block = squashing_transform->add(new_block)) { - ExpressionActions do_convert = ExpressionActions(converter); - do_convert.execute(new_block); + bool has_part = blockToPart(add_block); + if (has_part && merge_after_insert) + checkAndMerge(); } +} +bool SparkMergeTreeWriter::blockToPart(Block & block) +{ auto blocks_with_partition - = MergeTreeDataWriter::splitBlockIntoParts(squashing_transform->add(new_block), 10, metadata_snapshot, context); + = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), 10, metadata_snapshot, context); + + if (blocks_with_partition.empty()) + return false; + for (auto & item : blocks_with_partition) { size_t before_write_memory = 0; @@ -120,8 +145,7 @@ void SparkMergeTreeWriter::write(DB::Block & block) item.partition.clear(); } - if (!blocks_with_partition.empty() && merge_after_insert) - checkAndMerge(); + return true; } void SparkMergeTreeWriter::manualFreeMemory(size_t before_write_memory) @@ -130,10 +154,10 @@ void SparkMergeTreeWriter::manualFreeMemory(size_t before_write_memory) // it may alloc memory in current thread, and free on global thread. // Now, wo have not idea to clear global memory by used spark thread tracker. // So we manually correct the memory usage. - auto disk = storage->getStoragePolicy()->getAnyDisk(); - if (!disk->isRemote()) + if (!isRemoteStorage) return; + auto disk = storage->getStoragePolicy()->getAnyDisk(); std::lock_guard lock(memory_mutex); auto * memory_tracker = CurrentThread::getMemoryTracker(); if (memory_tracker && CurrentMemoryTracker::before_free) @@ -146,7 +170,6 @@ void SparkMergeTreeWriter::manualFreeMemory(size_t before_write_memory) memory_tracker->adjustWithUntrackedMemory(diff_ch_alloc); } - const size_t a = memory_tracker->get(); const size_t spark_alloc = CurrentMemoryTracker::current_memory(); const size_t diff_alloc = spark_alloc - memory_tracker->get(); @@ -157,38 +180,86 @@ void SparkMergeTreeWriter::manualFreeMemory(size_t before_write_memory) void SparkMergeTreeWriter::finalize() { - auto block = squashing_transform->add({}); - if (block.rows()) + if (auto block = squashing_transform->add({})) + { + if (block.rows()) + blockToPart(block); + } + + if (merge_after_insert) + finalizeMerge(); + + commitPartToRemoteStorageIfNeeded(); + saveMetadata(); +} + +void SparkMergeTreeWriter::saveMetadata() +{ + if (!isRemoteStorage) + return; + + for (const auto & merge_tree_data_part : new_parts.unsafeGet()) { - auto blocks_with_partition = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), 10, metadata_snapshot, context); - for (auto & item : blocks_with_partition) + auto part = dest_storage->loadDataPartsWithNames({merge_tree_data_part->name}); + if (part.empty()) { - size_t before_write_memory = 0; - if (auto * memory_tracker = CurrentThread::getMemoryTracker()) - { - CurrentThread::flushUntrackedMemory(); - before_write_memory = memory_tracker->get(); - } - - new_parts.emplace_back(writeTempPartAndFinalize(item, metadata_snapshot).part); - part_num++; - manualFreeMemory(before_write_memory); - /// Reset earlier to free memory - item.block.clear(); - item.partition.clear(); + LOG_WARNING( + &Poco::Logger::get("SparkMergeTreeWriter"), + "Save metadata failed because dest storage load part name {} empty.", + merge_tree_data_part->name); + continue; } - } - SCOPE_EXIT({ - for (auto merge_tree_data_part : new_parts.unsafeGet()) - saveFileStatus( - *storage, context, merge_tree_data_part->name, const_cast(merge_tree_data_part->getDataPartStorage())); - }); + saveFileStatus( + *dest_storage, context, merge_tree_data_part->name, const_cast(part.at(0)->getDataPartStorage())); + } +} - if (!merge_after_insert) +void SparkMergeTreeWriter::commitPartToRemoteStorageIfNeeded() +{ + if (!isRemoteStorage) return; - // wait all merge task end and do final merge + LOG_DEBUG( + &Poco::Logger::get("SparkMergeTreeWriter"), "Begin upload to disk {}.", dest_storage->getStoragePolicy()->getAnyDisk()->getName()); + + auto read_settings = context->getReadSettings(); + auto write_settings = context->getWriteSettings(); + Stopwatch watch; + for (const auto & merge_tree_data_part : new_parts.unsafeGet()) + { + String local_relative_path = storage->getRelativeDataPath() + "/" + merge_tree_data_part->name; + String remote_relative_path = dest_storage->getRelativeDataPath() + "/" + merge_tree_data_part->name; + + storage->getStoragePolicy()->getAnyDisk()->copyDirectoryContent( + local_relative_path, + dest_storage->getStoragePolicy()->getAnyDisk(), + remote_relative_path, + read_settings, + write_settings, + nullptr); + LOG_DEBUG( + &Poco::Logger::get("SparkMergeTreeWriter"), + "Upload part {} to disk {} success.", + merge_tree_data_part->name, + dest_storage->getStoragePolicy()->getAnyDisk()->getName()); + } + watch.stop(); + LOG_INFO( + &Poco::Logger::get("SparkMergeTreeWriter"), + "Upload to disk {} finished, total elapsed {} ms", + dest_storage->getStoragePolicy()->getAnyDisk()->getName(), + watch.elapsedMilliseconds()); + StorageMergeTreeFactory::freeStorage(temp_storage->getStorageID()); + temp_storage->dropAllData(); + LOG_DEBUG( + &Poco::Logger::get("SparkMergeTreeWriter"), "Clean temp table {} success.", temp_storage->getStorageID().getFullNameNotQuoted()); +} + +void SparkMergeTreeWriter::finalizeMerge() +{ + LOG_DEBUG(&Poco::Logger::get("SparkMergeTreeWriter"), "Waiting all merge task end and do final merge"); + // waiting all merge task end and do final merge thread_pool.wait(); size_t before_merge_size; @@ -200,30 +271,32 @@ void SparkMergeTreeWriter::finalize() } while (before_merge_size != new_parts.size()); std::unordered_set final_parts; - for (auto merge_tree_data_part : new_parts.unsafeGet()) + for (const auto & merge_tree_data_part : new_parts.unsafeGet()) final_parts.emplace(merge_tree_data_part->name); - for (const auto & tmp_part : tmp_parts) + // default storage need clean temp. + if (!temp_storage) { - if (final_parts.contains(tmp_part)) - continue; + for (const auto & tmp_part : tmp_parts) + { + if (final_parts.contains(tmp_part)) + continue; - GlobalThreadPool::instance().scheduleOrThrow( - [&]() -> void - { - for (auto disk : storage->getDisks()) + GlobalThreadPool::instance().scheduleOrThrow( + [&]() -> void { - auto full_path = storage->getFullPathOnDisk(disk); - disk->removeRecursive(full_path + "/" + tmp_part); - } - }); + for (const auto & disk : storage->getDisks()) + { + auto full_path = storage->getFullPathOnDisk(disk); + disk->removeRecursive(full_path + "/" + tmp_part); + } + }); + } } } -DB::MergeTreeDataWriter::TemporaryPart -SparkMergeTreeWriter::writeTempPartAndFinalize( - DB::BlockWithPartition & block_with_partition, - const DB::StorageMetadataPtr & metadata_snapshot) +DB::MergeTreeDataWriter::TemporaryPart SparkMergeTreeWriter::writeTempPartAndFinalize( + DB::BlockWithPartition & block_with_partition, const DB::StorageMetadataPtr & metadata_snapshot) { MergeTreeDataWriter::TemporaryPart temp_part; writeTempPart(temp_part, block_with_partition, metadata_snapshot); @@ -231,8 +304,8 @@ SparkMergeTreeWriter::writeTempPartAndFinalize( return temp_part; } -void SparkMergeTreeWriter::writeTempPart(MergeTreeDataWriter::TemporaryPart & temp_part, - BlockWithPartition & block_with_partition, const StorageMetadataPtr & metadata_snapshot) +void SparkMergeTreeWriter::writeTempPart( + MergeTreeDataWriter::TemporaryPart & temp_part, BlockWithPartition & block_with_partition, const StorageMetadataPtr & metadata_snapshot) { Block & block = block_with_partition.block; @@ -251,21 +324,13 @@ void SparkMergeTreeWriter::writeTempPart(MergeTreeDataWriter::TemporaryPart & te std::string part_dir; if (!partition_dir.empty() && !bucket_dir.empty()) - { - part_dir = fmt::format("{}/{}/{}_{:03d}", partition_dir, bucket_dir, uuid, part_num); - } + part_dir = fmt::format("{}/{}/{}_{:03d}", partition_dir, bucket_dir, part_name_prefix, part_num); else if (!partition_dir.empty()) - { - part_dir = fmt::format("{}/{}_{:03d}", partition_dir, uuid, part_num); - } + part_dir = fmt::format("{}/{}_{:03d}", partition_dir, part_name_prefix, part_num); else if (!bucket_dir.empty()) - { - part_dir = fmt::format("{}/{}_{:03d}", bucket_dir, uuid, part_num); - } + part_dir = fmt::format("{}/{}_{:03d}", bucket_dir, part_name_prefix, part_num); else - { - part_dir = fmt::format("{}_{:03d}", uuid, part_num); - } + part_dir = fmt::format("{}_{:03d}", part_name_prefix, part_num); String part_name = part_dir; @@ -369,8 +434,7 @@ void SparkMergeTreeWriter::writeTempPart(MergeTreeDataWriter::TemporaryPart & te auto finalizer = out->finalizePartAsync(new_data_part, data_settings->fsync_after_insert, nullptr, nullptr); temp_part.part = new_data_part; - temp_part.streams.emplace_back( - MergeTreeDataWriter::TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); + temp_part.streams.emplace_back(MergeTreeDataWriter::TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); } std::vector SparkMergeTreeWriter::getAllPartInfo() @@ -378,12 +442,11 @@ std::vector SparkMergeTreeWriter::getAllPartInfo() std::vector res; res.reserve(new_parts.size()); - for (auto part : new_parts.unsafeGet()) + for (const auto & part : new_parts.unsafeGet()) { res.emplace_back( PartInfo{part->name, part->getMarksCount(), part->getBytesOnDisk(), part->rows_count, partition_values, bucket_dir}); } - return res; } @@ -425,37 +488,42 @@ void SparkMergeTreeWriter::checkAndMerge(bool force) if (!force && new_parts.size() < merge_limit_parts) return; - auto doTask = [this]( - const ThreadGroupPtr & thread_group, - const std::vector prepare_merge_parts, - CustomStorageMergeTreePtr & storage, - String & partition_dir, - String & bucket_dir) -> std::vector + auto doMergeTask = [this](const std::vector & prepare_merge_parts) { - setThreadName("InsertWithMerge"); - ThreadStatus thread_status; - thread_status.attachToGroup(thread_group); - - size_t before_size = 0; - size_t after_size = 0; - for (const auto & prepare_merge_part : prepare_merge_parts) - before_size += prepare_merge_part->getBytesOnDisk(); - - std::unordered_map partition_values; - auto merged_parts - = mergeParts(prepare_merge_parts, partition_values, toString(UUIDHelpers::generateV4()), storage, partition_dir, bucket_dir); - for (const auto & merge_tree_data_part : merged_parts) - after_size += merge_tree_data_part->getBytesOnDisk(); - - LOG_DEBUG( - &Poco::Logger::get("SparkMergeTreeWriter"), - "Mergetree merge on insert finished, before merge part size {}, part count {}, after part size {}, part count {}.", - before_size, - prepare_merge_parts.size(), - after_size, - merged_parts.size()); + for (const auto & selected_part : prepare_merge_parts) + tmp_parts.emplace(selected_part->name); - return merged_parts; + thread_pool.scheduleOrThrow( + [this, prepare_merge_parts, thread_group = CurrentThread::getGroup()]() -> void + { + Stopwatch watch; + setThreadName("InsertWithMerge"); + ThreadStatus thread_status; + thread_status.attachToGroup(thread_group); + + size_t before_size = 0; + size_t after_size = 0; + for (const auto & prepare_merge_part : prepare_merge_parts) + before_size += prepare_merge_part->getBytesOnDisk(); + + std::unordered_map partition_values; + const auto merged_parts = mergeParts( + prepare_merge_parts, partition_values, toString(UUIDHelpers::generateV4()), storage, partition_dir, bucket_dir); + for (const auto & merge_tree_data_part : merged_parts) + after_size += merge_tree_data_part->getBytesOnDisk(); + + new_parts.emplace_back(merged_parts); + watch.stop(); + LOG_INFO( + &Poco::Logger::get("SparkMergeTreeWriter"), + "Merge success. Before merge part size {}, part count {}, after part size {}, part count {}, " + "total elapsed {} ms", + before_size, + prepare_merge_parts.size(), + after_size, + merged_parts.size(), + watch.elapsedMilliseconds()); + }); }; std::vector selected_parts; @@ -477,13 +545,7 @@ void SparkMergeTreeWriter::checkAndMerge(bool force) if (merge_min_size > totol_size && merge_limit_parts > selected_parts.size()) continue; - for (auto selected_part : selected_parts) - { - tmp_parts.emplace(selected_part->name); - } - - thread_pool.scheduleOrThrow([this, doTask, selected_parts, thread_group = CurrentThread::getGroup()]() -> void - { new_parts.emplace_back(doTask(thread_group, selected_parts, storage, partition_dir, bucket_dir)); }); + doMergeTask(selected_parts); selected_parts.clear(); totol_size = 0; } @@ -491,13 +553,7 @@ void SparkMergeTreeWriter::checkAndMerge(bool force) if (!selected_parts.empty()) { if (force && selected_parts.size() > 1) - { - for (auto selected_part : selected_parts) - tmp_parts.emplace(selected_part->name); - thread_pool.scheduleOrThrow( - [this, doTask, selected_parts, thread_group = CurrentThread::getGroup()]() -> void - { new_parts.emplace_back(doTask(thread_group, selected_parts, storage, partition_dir, bucket_dir)); }); - } + doMergeTask(selected_parts); else new_parts.emplace_back(selected_parts); } diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h index 5251d4cc447a..5c4b66403303 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h @@ -15,13 +15,15 @@ * limitations under the License. */ #pragma once + #include #include -#include #include +#include #include #include #include +#include namespace DB { @@ -51,10 +53,9 @@ class SparkMergeTreeWriter public: static String partInfosToJson(const std::vector & part_infos); SparkMergeTreeWriter( - CustomStorageMergeTreePtr storage_, - const DB::StorageMetadataPtr & metadata_snapshot_, + const MergeTreeTable & merge_tree_table, const DB::ContextPtr & context_, - const String & uuid_, + const String & part_name_prefix_, const String & partition_dir_ = "", const String & bucket_dir_ = ""); @@ -71,12 +72,20 @@ class SparkMergeTreeWriter void safeEmplaceBackPart(DB::MergeTreeDataPartPtr); void safeAddPart(DB::MergeTreeDataPartPtr); void manualFreeMemory(size_t before_write_memory); + void saveMetadata(); + void commitPartToRemoteStorageIfNeeded(); + void finalizeMerge(); + bool blockToPart(Block & block); - String uuid; + CustomStorageMergeTreePtr storage = nullptr; + CustomStorageMergeTreePtr dest_storage = nullptr; + CustomStorageMergeTreePtr temp_storage = nullptr; + DB::StorageMetadataPtr metadata_snapshot = nullptr; + + String part_name_prefix; String partition_dir; String bucket_dir; - CustomStorageMergeTreePtr storage; - DB::StorageMetadataPtr metadata_snapshot; + DB::ContextPtr context; std::unique_ptr squashing_transform; int part_num = 1; @@ -89,6 +98,7 @@ class SparkMergeTreeWriter size_t merge_min_size = 1024 * 1024 * 1024; size_t merge_limit_parts = 10; std::mutex memory_mutex; + bool isRemoteStorage = false; }; } diff --git a/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.cpp b/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.cpp index bb8d9d3d7140..0731ac92cd07 100644 --- a/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.cpp +++ b/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.cpp @@ -18,71 +18,56 @@ namespace local_engine { + +String StorageMergeTreeFactory::getTableName(const StorageID & id, const String & snapshot_id) +{ + auto table_name = id.database_name + "." + id.table_name; + // for optimize table + if (!snapshot_id.empty()) + table_name += "_" + snapshot_id; + + return table_name; +} + + StorageMergeTreeFactory & StorageMergeTreeFactory::instance() { static StorageMergeTreeFactory ret; return ret; } -void StorageMergeTreeFactory::freeStorage(StorageID id) +void StorageMergeTreeFactory::freeStorage(const StorageID & id, const String & snapshot_id) { - if (!id.hasUUID()) - { - return; - } - auto table_name = id.database_name + "." + id.table_name + "@" + toString(id.uuid); + auto table_name = getTableName(id, snapshot_id); { std::lock_guard lock(storage_map_mutex); if (storage_map->has(table_name)) - { storage_map->remove(table_name); - } } { std::lock_guard lock(datapart_mutex); if (datapart_map->has(table_name)) - { datapart_map->remove(table_name); - } } } CustomStorageMergeTreePtr -StorageMergeTreeFactory::getStorage(StorageID id, const String & snapshot_id, ColumnsDescription columns, std::function creator) +StorageMergeTreeFactory::getStorage(StorageID id, const String & snapshot_id, std::function creator) { - auto table_name = id.database_name + "." + id.table_name; - // for optimize table - if (id.hasUUID()) - { - table_name += "@" + toString(id.uuid); - } - else - { - table_name += "_" + snapshot_id; - } + auto table_name = getTableName(id, snapshot_id); std::lock_guard lock(storage_map_mutex); if (!storage_map->has(table_name)) - { storage_map->add(table_name, creator()); - } return *(storage_map->get(table_name)); } -DataPartsVector StorageMergeTreeFactory::getDataParts(StorageID id, const String & snapshot_id, std::unordered_set part_name) +DataPartsVector StorageMergeTreeFactory::getDataPartsByNames(const StorageID & id, const String & snapshot_id, std::unordered_set part_name) { DataPartsVector res; - auto table_name = id.database_name + "." + id.table_name; - // for optimize table - if (id.hasUUID()) - { - table_name += "@" + toString(id.uuid); - } - else - { - table_name += "_" + snapshot_id; - } + auto table_name = getTableName(id, snapshot_id); + std::lock_guard lock(datapart_mutex); std::unordered_set missing_names; if (!datapart_map->has(table_name)) [[unlikely]] diff --git a/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.h b/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.h index 82dae3745211..d7bcb93c07d7 100644 --- a/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.h +++ b/cpp-ch/local-engine/Storages/StorageMergeTreeFactory.h @@ -28,10 +28,10 @@ class StorageMergeTreeFactory { public: static StorageMergeTreeFactory & instance(); - static void freeStorage(StorageID id); + static void freeStorage(const StorageID & id, const String & snapshot_id = ""); static CustomStorageMergeTreePtr - getStorage(StorageID id, const String & snapshot_id, ColumnsDescription columns, std::function creator); - static DataPartsVector getDataParts(StorageID id, const String & snapshot_id, std::unordered_set part_name); + getStorage(StorageID id, const String & snapshot_id, std::function creator); + static DataPartsVector getDataPartsByNames(const StorageID & id, const String & snapshot_id, std::unordered_set part_name); static void init_cache_map() { auto & storage_map_v = storage_map; @@ -61,6 +61,8 @@ class StorageMergeTreeFactory if (datapart_map) datapart_map->clear(); } + static String getTableName(const StorageID & id, const String & snapshot_id); + private: static std::unique_ptr> storage_map; static std::unique_ptr>>> datapart_map; diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 00a7b1b0ad82..1c088720dbcb 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -1059,11 +1059,10 @@ JNIEXPORT jlong Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniW substrait::ReadRel::ExtensionTable extension_table = local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); - auto storage = local_engine::MergeTreeRelParser::parseStorage( - extension_table, local_engine::SerializedPlanParser::global_context); + auto merge_tree_table = local_engine::MergeTreeRelParser::parseMergeTreeTable(extension_table); auto uuid = uuid_str + "_" + task_id; auto * writer = new local_engine::SparkMergeTreeWriter( - storage, storage->getInMemoryMetadataPtr(), query_context, uuid, partition_dir, bucket_dir); + merge_tree_table, query_context, uuid, partition_dir, bucket_dir); env->ReleaseByteArrayElements(plan_, plan_buf_addr, JNI_ABORT); env->ReleaseByteArrayElements(split_info_, split_info_addr, JNI_ABORT); @@ -1196,28 +1195,27 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn substrait::ReadRel::ExtensionTable extension_table = local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); - - UUID uuid = UUIDHelpers::generateV4(); // each task using its own CustomStorageMergeTree, don't reuse - auto storage = local_engine::MergeTreeRelParser::parseStorage( - extension_table, local_engine::SerializedPlanParser::global_context, uuid); - google::protobuf::StringValue table; table.ParseFromString(extension_table.detail().value()); auto merge_tree_table = local_engine::parseMergeTreeTableString(table.value()); - DB::StorageID table_id(merge_tree_table.database, merge_tree_table.table, uuid); - local_engine::TempStorageFreer freer {table_id}; // to release temp CustomStorageMergeTree with RAII - auto storage_factory = local_engine::StorageMergeTreeFactory::instance(); - std::vector selected_parts = storage_factory.getDataParts(table_id, merge_tree_table.snapshot_id, merge_tree_table.getPartNames()); + + // each task using its own CustomStorageMergeTree, don't reuse + auto temp_storage + = local_engine::MergeTreeRelParser::copyToVirtualStorage(merge_tree_table, local_engine::SerializedPlanParser::global_context); + + local_engine::TempStorageFreer freer{temp_storage->getStorageID()}; // to release temp CustomStorageMergeTree with RAII + std::vector selected_parts + = local_engine::StorageMergeTreeFactory::instance().getDataPartsByNames(temp_storage->getStorageID(), "", merge_tree_table.getPartNames()); std::unordered_map partition_values; std::vector loaded = - local_engine::mergeParts(selected_parts, partition_values, uuid_str, storage, partition_dir, bucket_dir); + local_engine::mergeParts(selected_parts, partition_values, uuid_str, temp_storage, partition_dir, bucket_dir); std::vector res; for (auto & partPtr : loaded) { saveFileStatus( - *storage, + *temp_storage, local_engine::SerializedPlanParser::global_context, partPtr->name, const_cast(partPtr->getDataPartStorage())); diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala index 859cca842df3..f9ad5201d8db 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala @@ -24,7 +24,7 @@ import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OrderPreservingUnaryNode} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution._ @@ -65,6 +65,11 @@ case class FakeRowAdaptor(child: SparkPlan) doExecuteColumnar().map(cb => new FakeRow(cb)) } + override def outputOrdering: Seq[SortOrder] = child match { + case aqe: AdaptiveSparkPlanExec => aqe.executedPlan.outputOrdering + case _ => child.outputOrdering + } + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { if (child.supportsColumnar) { child.executeColumnar() From 6cb8f7cdea4fe02677698b947f7189a9a0c75cb0 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 30 May 2024 21:18:17 +0800 Subject: [PATCH 179/402] [VL] Daily Update Velox Version (2024_05_30) (#5919) Co-authored-by: PHILO-HE --- cpp/velox/memory/VeloxMemoryManager.cc | 12 ++++++------ ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index b7bd3a9f9a58..496ebf4522e1 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -44,7 +44,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { uint64_t growCapacity(velox::memory::MemoryPool* pool, uint64_t targetBytes) override { std::lock_guard l(mutex_); listener_->allocationChanged(targetBytes); - if (!pool->grow(targetBytes, 0)) { + if (!growPool(pool, targetBytes, 0)) { VELOX_FAIL("Failed to grow root pool's capacity for {}", velox::succinctBytes(targetBytes)); } return targetBytes; @@ -81,7 +81,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { auto pool = pools.at(0); const uint64_t oldCapacity = pool->capacity(); pool->reclaim(targetBytes, 0, status); // ignore the output - pool->shrink(0); + shrinkPool(pool.get(), 0); const uint64_t newCapacity = pool->capacity(); uint64_t total = oldCapacity - newCapacity; listener_->allocationChanged(-total); @@ -104,14 +104,14 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { // We should pass bytes as parameter "reservationBytes" when calling ::grow. auto freeByes = pool->freeBytes(); if (freeByes > bytes) { - if (pool->grow(0, bytes)) { + if (growPool(pool, 0, bytes)) { return; } } - auto reclaimedFreeBytes = pool->shrink(0); + auto reclaimedFreeBytes = shrinkPool(pool, 0); auto neededBytes = velox::bits::roundUp(bytes - reclaimedFreeBytes, memoryPoolTransferCapacity_); listener_->allocationChanged(neededBytes); - auto ret = pool->grow(reclaimedFreeBytes + neededBytes, bytes); + auto ret = growPool(pool, reclaimedFreeBytes + neededBytes, bytes); VELOX_CHECK( ret, "{} failed to grow {} bytes, current state {}", @@ -121,7 +121,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { } uint64_t shrinkCapacityLocked(velox::memory::MemoryPool* pool, uint64_t bytes) { - uint64_t freeBytes = pool->shrink(bytes); + uint64_t freeBytes = shrinkPool(pool, bytes); listener_->allocationChanged(-freeBytes); return freeBytes; } diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 9fe130c1f69a..ff31daddb7b4 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_29 +VELOX_BRANCH=2024_05_30 VELOX_HOME="" #Set on run gluten on HDFS From c7aaee250a5319bdd1e109b43d3244a191f5ae68 Mon Sep 17 00:00:00 2001 From: Jaime Pan <33685703+NEUpanning@users.noreply.github.com> Date: Thu, 30 May 2024 22:35:58 +0800 Subject: [PATCH 180/402] [GLUTEN-5701][VL] Add overflow test case for from_unixtime function (#5894) --- .../sql/catalyst/expressions/GlutenDateExpressionsSuite.scala | 3 +++ .../sql/catalyst/expressions/GlutenDateExpressionsSuite.scala | 3 +++ .../sql/catalyst/expressions/GlutenDateExpressionsSuite.scala | 3 +++ .../sql/catalyst/expressions/GlutenDateExpressionsSuite.scala | 3 +++ 4 files changed, 12 insertions(+) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 29b9141b7200..44d4502aedac 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -382,6 +382,9 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr checkEvaluation( FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId), sdf2.format(new Timestamp(-1000000))) + checkEvaluation( + FromUnixTime(Literal(Long.MaxValue), Literal(fmt2), timeZoneId), + sdf2.format(new Timestamp(-1000))) checkEvaluation( FromUnixTime( Literal.create(null, LongType), diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 701dd1e4a826..234537feef8a 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -380,6 +380,9 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr checkEvaluation( FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId), sdf2.format(new Timestamp(-1000000))) + checkEvaluation( + FromUnixTime(Literal(Long.MaxValue), Literal(fmt2), timeZoneId), + sdf2.format(new Timestamp(-1000))) checkEvaluation( FromUnixTime( Literal.create(null, LongType), diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index e726dcea18c7..f4a9dfd56a92 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -380,6 +380,9 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr checkEvaluation( FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId), sdf2.format(new Timestamp(-1000000))) + checkEvaluation( + FromUnixTime(Literal(Long.MaxValue), Literal(fmt2), timeZoneId), + sdf2.format(new Timestamp(-1000))) checkEvaluation( FromUnixTime( Literal.create(null, LongType), diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index e726dcea18c7..f4a9dfd56a92 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -380,6 +380,9 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr checkEvaluation( FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId), sdf2.format(new Timestamp(-1000000))) + checkEvaluation( + FromUnixTime(Literal(Long.MaxValue), Literal(fmt2), timeZoneId), + sdf2.format(new Timestamp(-1000))) checkEvaluation( FromUnixTime( Literal.create(null, LongType), From 73eb21db45a83c3180518bfcef3c5d343595f452 Mon Sep 17 00:00:00 2001 From: lgbo Date: Thu, 30 May 2024 23:35:59 +0800 Subject: [PATCH 181/402] [GLUTEN-5904][CH] Convert nan to null which comes from stddev (#5913) [CH] Convert nan to null which comes from stddev --- .../resources/tpcds-queries-output/q17.out | 4 +- .../execution/GlutenClickHouseTPCHSuite.scala | 15 ++++ .../CommonAggregateFunctionParser.cpp | 2 - .../SimpleStatisticsFunctions.cpp | 70 +++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/aggregate_function_parser/SimpleStatisticsFunctions.cpp diff --git a/backends-clickhouse/src/test/resources/tpcds-queries-output/q17.out b/backends-clickhouse/src/test/resources/tpcds-queries-output/q17.out index ab9c217f7d73..95891dd39098 100644 --- a/backends-clickhouse/src/test/resources/tpcds-queries-output/q17.out +++ b/backends-clickhouse/src/test/resources/tpcds-queries-output/q17.out @@ -1,2 +1,2 @@ -AAAAAAAADOCAAAAA|-|Little, national services will buy young molecules. In part video-taped activities join now|-|TN|-|1|-|24.0|-|NaN|-|NaN|-|1|-|11.0|-|NaN|-|NaN|-|1|-|49.0|-|NaN|-|NaN -AAAAAAAAEBOBAAAA|-|Special words should tell by a follower|-|TN|-|1|-|66.0|-|NaN|-|NaN|-|1|-|38.0|-|NaN|-|NaN|-|1|-|56.0|-|NaN|-|NaN +AAAAAAAADOCAAAAA|-|Little, national services will buy young molecules. In part video-taped activities join now|-|TN|-|1|-|24.0|-|null|-|null|-|1|-|11.0|-|null|-|null|-|1|-|49.0|-|null|-|null +AAAAAAAAEBOBAAAA|-|Special words should tell by a follower|-|TN|-|1|-|66.0|-|null|-|null|-|1|-|38.0|-|null|-|null|-|1|-|56.0|-|null|-|null diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala index 6dcb83de4740..96b2cb09b163 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala @@ -481,5 +481,20 @@ class GlutenClickHouseTPCHSuite extends GlutenClickHouseTPCHAbstractSuite { spark.sql(table_drop_sql) } + + test("GLUTEN-5904 NaN values from stddev") { + val sql1 = + """ + |select a, stddev(b/c) from (select * from values (1,2, 1), (1,3,0) as data(a,b,c)) + |group by a + |""".stripMargin + compareResultsAgainstVanillaSpark(sql1, true, { _ => }) + val sql2 = + """ + |select a, stddev(b) from (select * from values (1,2, 1) as data(a,b,c)) group by a + |""".stripMargin + compareResultsAgainstVanillaSpark(sql2, true, { _ => }) + + } } // scalastyle:off line.size.limit diff --git a/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp b/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp index afe65f7931a6..1619c74106d1 100644 --- a/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp +++ b/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp @@ -25,8 +25,6 @@ REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(Sum, sum, sum) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(Avg, avg, avg) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(Min, min, min) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(Max, max, max) -REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(StdDev, stddev, stddev_samp) -REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(StdDevSamp, stddev_samp, stddev_samp) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(StdDevPop, stddev_pop, stddev_pop) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(BitAnd, bit_and, groupBitAnd) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(BitOr, bit_or, groupBitOr) diff --git a/cpp-ch/local-engine/Parser/aggregate_function_parser/SimpleStatisticsFunctions.cpp b/cpp-ch/local-engine/Parser/aggregate_function_parser/SimpleStatisticsFunctions.cpp new file mode 100644 index 000000000000..7e75e20bb742 --- /dev/null +++ b/cpp-ch/local-engine/Parser/aggregate_function_parser/SimpleStatisticsFunctions.cpp @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +namespace local_engine +{ +/// For stddev +struct StddevNameStruct +{ + static constexpr auto spark_name = "stddev"; + static constexpr auto ch_name = "stddev"; +}; + +struct StddevSampNameStruct +{ + static constexpr auto spark_name = "stddev_samp"; + static constexpr auto ch_name = "stddev_samp"; +}; +template +class AggregateFunctionParserStddev final : public AggregateFunctionParser +{ +public: + AggregateFunctionParserStddev(SerializedPlanParser * plan_parser_) : AggregateFunctionParser(plan_parser_) { } + ~AggregateFunctionParserStddev() override = default; + String getName() const override { return NameStruct::spark_name; } + static constexpr auto name = NameStruct::spark_name; + String getCHFunctionName(const CommonFunctionInfo &) const override { return NameStruct::ch_name; } + String getCHFunctionName(DB::DataTypes &) const override { return NameStruct::ch_name; } + const DB::ActionsDAG::Node * convertNodeTypeIfNeeded( + const CommonFunctionInfo & func_info, + const DB::ActionsDAG::Node * func_node, + DB::ActionsDAGPtr & actions_dag, + bool with_nullability) const override + { + /// result is nullable. + /// if result is NaN, convert it to NULL. + auto is_nan_func_node = toFunctionNode(actions_dag, "isNaN", getUniqueName("isNaN"), {func_node}); + auto null_type = DB::makeNullable(func_node->result_type); + auto nullable_col = null_type->createColumn(); + nullable_col->insertDefault(); + const auto * null_node + = &actions_dag->addColumn(DB::ColumnWithTypeAndName(std::move(nullable_col), null_type, getUniqueName("null"))); + DB::ActionsDAG::NodeRawConstPtrs convert_nan_func_args = {is_nan_func_node, null_node, func_node}; + + func_node = toFunctionNode(actions_dag, "if", func_node->result_name, convert_nan_func_args); + actions_dag->addOrReplaceInOutputs(*func_node); + return func_node; + } +}; + +static const AggregateFunctionParserRegister> registerer_stddev; +static const AggregateFunctionParserRegister> registerer_stddev_samp; +} From 4c3fbb8e369de480ba31b6385eb35c6d8c3852be Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Fri, 31 May 2024 09:19:25 +0800 Subject: [PATCH 182/402] Fix shuffle with round robin partitioning fail (#5928) --- .../velox/VeloxSparkPlanExecApi.scala | 48 ++++++++++++------- .../gluten/execution/TestOperator.scala | 23 +++++++++ 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 92be63a583f6..58b27e8a7f80 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -347,23 +347,39 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { // See https://github.com/apache/spark/blob/609bd4839e5d504917de74ed1cb9c23645fba51f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala#L279-L283 // scalastyle:on line.size.limit allowHashOnMap { - val hashExpr = new Murmur3Hash(newChild.output) - val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ newChild.output - val projectTransformer = ProjectExecTransformer(projectList, newChild) - val sortOrder = SortOrder(projectTransformer.output.head, Ascending) - val sortByHashCode = - SortExecTransformer(Seq(sortOrder), global = false, projectTransformer) - val dropSortColumnTransformer = - ProjectExecTransformer(projectList.drop(1), sortByHashCode) - val validationResult = dropSortColumnTransformer.doValidate() - if (validationResult.isValid) { - ColumnarShuffleExchangeExec( - shuffle, - dropSortColumnTransformer, - dropSortColumnTransformer.output) + // Velox hash expression does not support null type and we also do not need to sort + // null type since the value always be null. + val columnsForHash = newChild.output.filterNot(_.dataType == NullType) + if (columnsForHash.isEmpty) { + ColumnarShuffleExchangeExec(shuffle, newChild, newChild.output) } else { - TransformHints.tagNotTransformable(shuffle, validationResult) - shuffle.withNewChildren(newChild :: Nil) + val hashExpr = new Murmur3Hash(columnsForHash) + val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ newChild.output + val projectTransformer = ProjectExecTransformer(projectList, newChild) + val projectBeforeSortValidationResult = projectTransformer.doValidate() + // Make sure we support offload hash expression + val projectBeforeSort = if (projectBeforeSortValidationResult.isValid) { + projectTransformer + } else { + val project = ProjectExec(projectList, newChild) + TransformHints.tagNotTransformable(project, projectBeforeSortValidationResult) + project + } + val sortOrder = SortOrder(projectBeforeSort.output.head, Ascending) + val sortByHashCode = + SortExecTransformer(Seq(sortOrder), global = false, projectBeforeSort) + val dropSortColumnTransformer = + ProjectExecTransformer(projectList.drop(1), sortByHashCode) + val validationResult = dropSortColumnTransformer.doValidate() + if (validationResult.isValid) { + ColumnarShuffleExchangeExec( + shuffle, + dropSortColumnTransformer, + dropSortColumnTransformer.output) + } else { + TransformHints.tagNotTransformable(shuffle, validationResult) + shuffle.withNewChildren(newChild :: Nil) + } } } case _ => diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 7bbc24d45b6d..7088b7b072d9 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1536,4 +1536,27 @@ class TestOperator extends VeloxWholeStageTransformerSuite { checkGlutenOperatorMatch[GenerateExecTransformer] } } + + test("Fix shuffle with round robin partitioning fail") { + def checkNullTypeRepartition(df: => DataFrame, numProject: Int): Unit = { + var expected: Array[Row] = null + withSQLConf("spark.sql.execution.sortBeforeRepartition" -> "false") { + expected = df.collect() + } + val actual = df + checkAnswer(actual, expected) + assert( + collect(actual.queryExecution.executedPlan) { case p: ProjectExec => p }.size == numProject + ) + } + + checkNullTypeRepartition( + spark.table("lineitem").selectExpr("l_orderkey", "null as x").repartition(), + 0 + ) + checkNullTypeRepartition( + spark.table("lineitem").selectExpr("null as x", "null as y").repartition(), + 1 + ) + } } From 1a0b969250de57cf7cb6266484f755064058ce0e Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Fri, 31 May 2024 09:28:14 +0800 Subject: [PATCH 183/402] [CORE] Move driver/executor endpoint to CH backend (#5914) --- .../backendsapi/clickhouse/CHBackend.scala | 1 - .../clickhouse/CHBroadcastApi.scala | 45 ------------------- .../clickhouse/CHListenerApi.scala | 16 +++++-- .../clickhouse/CHTransformerApi.scala | 5 +++ .../execution/CHHashJoinExecTransformer.scala | 20 +++++++-- .../CHGlutenSQLAppStatusListener.scala | 11 ++++- .../spark/rpc/GlutenDriverEndpoint.scala | 0 .../spark/rpc/GlutenExecutorEndpoint.scala | 11 +++-- .../apache/spark/rpc/GlutenRpcConstants.scala | 0 .../apache/spark/rpc/GlutenRpcMessages.scala | 0 .../backendsapi/velox/VeloxBackend.scala | 1 - .../backendsapi/velox/VeloxBroadcastApi.scala | 32 ------------- .../backendsapi/velox/VeloxListenerApi.scala | 12 ++--- .../velox/VeloxSparkPlanExecApi.scala | 2 +- .../ShuffledHashJoinExecTransformer.scala | 9 +++- ...adcastNestedLoopJoinExecTransformer.scala} | 13 ++++-- .../gluten/utils/VeloxBloomFilterTest.java | 43 +++++++++++++++++- gluten-core/pom.xml | 4 -- .../org/apache/gluten/GlutenPlugin.scala | 8 +--- .../apache/gluten/backendsapi/Backend.scala | 2 - .../backendsapi/BackendsApiManager.scala | 4 -- .../gluten/backendsapi/BroadcastApi.scala | 42 ----------------- .../gluten/backendsapi/ListenerApi.scala | 7 +-- .../gluten/backendsapi/TransformerApi.scala | 3 ++ ...oadcastNestedLoopJoinExecTransformer.scala | 18 +------- .../execution/JoinExecTransformer.scala | 16 +------ .../listener/GlutenListenerFactory.scala | 3 -- .../apache/spark/sql/GlutenQueryTest.scala | 4 +- 28 files changed, 131 insertions(+), 201 deletions(-) delete mode 100644 backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBroadcastApi.scala rename gluten-core/src/main/scala/org/apache/spark/listener/GlutenSQLAppStatusListener.scala => backends-clickhouse/src/main/scala/org/apache/spark/listener/CHGlutenSQLAppStatusListener.scala (86%) rename {gluten-core => backends-clickhouse}/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala (100%) rename {gluten-core => backends-clickhouse}/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala (89%) rename {gluten-core => backends-clickhouse}/src/main/scala/org/apache/spark/rpc/GlutenRpcConstants.scala (100%) rename {gluten-core => backends-clickhouse}/src/main/scala/org/apache/spark/rpc/GlutenRpcMessages.scala (100%) delete mode 100644 backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBroadcastApi.scala rename backends-velox/src/main/scala/org/apache/gluten/execution/{GlutenBroadcastNestedLoopJoinExecTransformer.scala => VeloxBroadcastNestedLoopJoinExecTransformer.scala} (76%) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/backendsapi/BroadcastApi.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index e5f68a8691ba..c79d0aaee800 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -49,7 +49,6 @@ class CHBackend extends Backend { override def validatorApi(): ValidatorApi = new CHValidatorApi override def metricsApi(): MetricsApi = new CHMetricsApi override def listenerApi(): ListenerApi = new CHListenerApi - override def broadcastApi(): BroadcastApi = new CHBroadcastApi override def settings(): BackendSettingsApi = CHBackendSettings } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBroadcastApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBroadcastApi.scala deleted file mode 100644 index d70ba6b8d2f1..000000000000 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBroadcastApi.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.backendsapi.clickhouse - -import org.apache.gluten.backendsapi.BroadcastApi -import org.apache.gluten.execution.CHBroadcastBuildSideCache - -import org.apache.spark.internal.Logging -import org.apache.spark.rpc.GlutenDriverEndpoint - -class CHBroadcastApi extends BroadcastApi with Logging { - override def cleanExecutionBroadcastTable( - executionId: String, - broadcastTableIds: java.util.Set[String]): Unit = { - if (broadcastTableIds != null) { - broadcastTableIds.forEach( - resource_id => CHBroadcastBuildSideCache.invalidateBroadcastHashtable(resource_id)) - } - } - - override def collectExecutionBroadcastTableId(executionId: String, buildTableId: String): Unit = { - if (executionId != null) { - GlutenDriverEndpoint.collectResources(executionId, buildTableId) - } else { - logWarning( - s"Can't not trace broadcast hash table data $buildTableId" + - s" because execution id is null." + - s" Will clean up until expire time.") - } - } -} diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala index 952812d68c2a..665fdba88e55 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala @@ -23,9 +23,12 @@ import org.apache.gluten.execution.datasource.{GlutenOrcWriterInjects, GlutenPar import org.apache.gluten.expression.UDFMappings import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, JniLibLoader} -import org.apache.spark.SparkConf +import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.api.plugin.PluginContext import org.apache.spark.internal.Logging +import org.apache.spark.listener.CHGlutenSQLAppStatusListener import org.apache.spark.network.util.JavaUtils +import org.apache.spark.rpc.{GlutenDriverEndpoint, GlutenExecutorEndpoint} import org.apache.spark.sql.execution.datasources.v1._ import org.apache.spark.util.SparkDirectoryUtil @@ -35,11 +38,18 @@ import java.util.TimeZone class CHListenerApi extends ListenerApi with Logging { - override def onDriverStart(conf: SparkConf): Unit = initialize(conf, isDriver = true) + override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = { + GlutenDriverEndpoint.glutenDriverEndpointRef = (new GlutenDriverEndpoint).self + CHGlutenSQLAppStatusListener.registerListener(sc) + initialize(pc.conf, isDriver = true) + } override def onDriverShutdown(): Unit = shutdown() - override def onExecutorStart(conf: SparkConf): Unit = initialize(conf, isDriver = false) + override def onExecutorStart(pc: PluginContext): Unit = { + GlutenExecutorEndpoint.executorEndpoint = new GlutenExecutorEndpoint(pc.executorID, pc.conf) + initialize(pc.conf, isDriver = false) + } override def onExecutorShutdown(): Unit = shutdown() diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index ea3398e77dfa..0aab14b78c87 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -23,6 +23,7 @@ import org.apache.gluten.substrait.expression.{BooleanLiteralNode, ExpressionBui import org.apache.gluten.utils.{CHInputPartitionsUtil, ExpressionDocUtil} import org.apache.spark.internal.Logging +import org.apache.spark.rpc.GlutenDriverEndpoint import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 @@ -227,4 +228,8 @@ class CHTransformerApi extends TransformerApi with Logging { } override def packPBMessage(message: Message): Any = Any.pack(message) + + override def invalidateSQLExecutionResource(executionId: String): Unit = { + GlutenDriverEndpoint.invalidateResourceRelation(executionId) + } } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala index 046f26043c5d..c3ab89df5bb1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala @@ -21,10 +21,12 @@ import org.apache.gluten.extension.ValidationResult import org.apache.gluten.utils.CHJoinValidateUtil import org.apache.spark.{broadcast, SparkContext} +import org.apache.spark.rdd.RDD +import org.apache.spark.rpc.GlutenDriverEndpoint import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.optimizer.BuildSide import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} import org.apache.spark.sql.execution.joins.BuildSideRelation import org.apache.spark.sql.vectorized.ColumnarBatch @@ -116,10 +118,22 @@ case class CHBroadcastHashJoinExecTransformer( super.doValidateInternal() } - override protected def createBroadcastBuildSideRDD(): BroadcastBuildSideRDD = { + override def columnarInputRDDs: Seq[RDD[ColumnarBatch]] = { + val streamedRDD = getColumnarInputRDDs(streamedPlan) + val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + if (executionId != null) { + GlutenDriverEndpoint.collectResources(executionId, buildHashTableId) + } else { + logWarning( + s"Can't not trace broadcast hash table data $buildHashTableId" + + s" because execution id is null." + + s" Will clean up until expire time.") + } val broadcast = buildPlan.executeBroadcast[BuildSideRelation]() val context = BroadCastHashJoinContext(buildKeyExprs, joinType, buildPlan.output, buildHashTableId) - CHBroadcastBuildSideRDD(sparkContext, broadcast, context) + val broadcastRDD = CHBroadcastBuildSideRDD(sparkContext, broadcast, context) + // FIXME: Do we have to make build side a RDD? + streamedRDD :+ broadcastRDD } } diff --git a/gluten-core/src/main/scala/org/apache/spark/listener/GlutenSQLAppStatusListener.scala b/backends-clickhouse/src/main/scala/org/apache/spark/listener/CHGlutenSQLAppStatusListener.scala similarity index 86% rename from gluten-core/src/main/scala/org/apache/spark/listener/GlutenSQLAppStatusListener.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/listener/CHGlutenSQLAppStatusListener.scala index 8c98d30442a5..7984fa846506 100644 --- a/gluten-core/src/main/scala/org/apache/spark/listener/GlutenSQLAppStatusListener.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/listener/CHGlutenSQLAppStatusListener.scala @@ -16,14 +16,15 @@ */ package org.apache.spark.listener +import org.apache.spark.SparkContext import org.apache.spark.internal.Logging +import org.apache.spark.rpc.{GlutenDriverEndpoint, RpcEndpointRef} import org.apache.spark.rpc.GlutenRpcMessages._ -import org.apache.spark.rpc.RpcEndpointRef import org.apache.spark.scheduler._ import org.apache.spark.sql.execution.ui._ /** Gluten SQL listener. Used for monitor sql on whole life cycle.Create and release resource. */ -class GlutenSQLAppStatusListener(val driverEndpointRef: RpcEndpointRef) +class CHGlutenSQLAppStatusListener(val driverEndpointRef: RpcEndpointRef) extends SparkListener with Logging { @@ -68,3 +69,9 @@ class GlutenSQLAppStatusListener(val driverEndpointRef: RpcEndpointRef) logTrace(s"Execution $executionId end.") } } +object CHGlutenSQLAppStatusListener { + def registerListener(sc: SparkContext): Unit = { + sc.listenerBus.addToStatusQueue( + new CHGlutenSQLAppStatusListener(GlutenDriverEndpoint.glutenDriverEndpointRef)) + } +} diff --git a/gluten-core/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala similarity index 100% rename from gluten-core/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala diff --git a/gluten-core/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala similarity index 89% rename from gluten-core/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala index e48194232771..f05933ef78e3 100644 --- a/gluten-core/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenExecutorEndpoint.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.rpc -import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.execution.CHBroadcastBuildSideCache import org.apache.spark.{SparkConf, SparkEnv} import org.apache.spark.internal.{config, Logging} @@ -60,10 +60,15 @@ class GlutenExecutorEndpoint(val executorId: String, val conf: SparkConf) override def receive: PartialFunction[Any, Unit] = { case GlutenCleanExecutionResource(executionId, hashIds) => - BackendsApiManager.getBroadcastApiInstance - .cleanExecutionBroadcastTable(executionId, hashIds) + if (executionId != null) { + hashIds.forEach( + resource_id => CHBroadcastBuildSideCache.invalidateBroadcastHashtable(resource_id)) + } case e => logError(s"Received unexpected message. $e") } } +object GlutenExecutorEndpoint { + var executorEndpoint: GlutenExecutorEndpoint = _ +} diff --git a/gluten-core/src/main/scala/org/apache/spark/rpc/GlutenRpcConstants.scala b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenRpcConstants.scala similarity index 100% rename from gluten-core/src/main/scala/org/apache/spark/rpc/GlutenRpcConstants.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenRpcConstants.scala diff --git a/gluten-core/src/main/scala/org/apache/spark/rpc/GlutenRpcMessages.scala b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenRpcMessages.scala similarity index 100% rename from gluten-core/src/main/scala/org/apache/spark/rpc/GlutenRpcMessages.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenRpcMessages.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 7f928bd330b7..c8dbfb29e485 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -51,7 +51,6 @@ class VeloxBackend extends Backend { override def validatorApi(): ValidatorApi = new VeloxValidatorApi override def metricsApi(): MetricsApi = new VeloxMetricsApi override def listenerApi(): ListenerApi = new VeloxListenerApi - override def broadcastApi(): BroadcastApi = new VeloxBroadcastApi override def settings(): BackendSettingsApi = VeloxBackendSettings } diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBroadcastApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBroadcastApi.scala deleted file mode 100644 index bae3bb63599e..000000000000 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBroadcastApi.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.backendsapi.velox - -import org.apache.gluten.backendsapi.BroadcastApi - -import java.util - -class VeloxBroadcastApi extends BroadcastApi { - - override def collectExecutionBroadcastTableId(executionId: String, buildTableId: String): Unit = - super.collectExecutionBroadcastTableId(executionId, buildTableId) - - override def cleanExecutionBroadcastTable( - executionId: String, - broadcastTableIds: util.Set[String]): Unit = - super.cleanExecutionBroadcastTable(executionId, broadcastTableIds) -} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index bbeb3a2715fe..0b1d131d318e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -25,7 +25,8 @@ import org.apache.gluten.init.NativeBackendInitializer import org.apache.gluten.utils._ import org.apache.gluten.vectorized.{JniLibLoader, JniWorkspace} -import org.apache.spark.SparkConf +import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.api.plugin.PluginContext import org.apache.spark.sql.execution.datasources.velox.{VeloxOrcWriterInjects, VeloxParquetWriterInjects, VeloxRowSplitter} import org.apache.spark.sql.expression.UDFResolver import org.apache.spark.sql.internal.{GlutenConfigUtil, StaticSQLConf} @@ -38,7 +39,8 @@ import scala.sys.process._ class VeloxListenerApi extends ListenerApi { private val ARROW_VERSION = "1500" - override def onDriverStart(conf: SparkConf): Unit = { + override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = { + val conf = pc.conf() // sql table cache serializer if (conf.getBoolean(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, defaultValue = false)) { conf.set( @@ -51,9 +53,9 @@ class VeloxListenerApi extends ListenerApi { override def onDriverShutdown(): Unit = shutdown() - override def onExecutorStart(conf: SparkConf): Unit = { - UDFResolver.resolveUdfConf(conf, isDriver = false) - initialize(conf) + override def onExecutorStart(pc: PluginContext): Unit = { + UDFResolver.resolveUdfConf(pc.conf(), isDriver = false) + initialize(pc.conf()) } override def onExecutorShutdown(): Unit = shutdown() diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 58b27e8a7f80..16c11f111abc 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -473,7 +473,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { buildSide: BuildSide, joinType: JoinType, condition: Option[Expression]): BroadcastNestedLoopJoinExecTransformer = - GlutenBroadcastNestedLoopJoinExecTransformer( + VeloxBroadcastNestedLoopJoinExecTransformer( left, right, buildSide, diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala index c9c60772fe73..002afea31624 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala @@ -16,12 +16,14 @@ */ package org.apache.gluten.execution +import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.execution.{FilterExec, SparkPlan} import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.execution.joins.BuildSideRelation +import org.apache.spark.sql.vectorized.ColumnarBatch import io.substrait.proto.JoinRel @@ -196,8 +198,11 @@ case class BroadcastHashJoinExecTransformer( newRight: SparkPlan): BroadcastHashJoinExecTransformer = copy(left = newLeft, right = newRight) - override protected def createBroadcastBuildSideRDD(): BroadcastBuildSideRDD = { + override def columnarInputRDDs: Seq[RDD[ColumnarBatch]] = { + val streamedRDD = getColumnarInputRDDs(streamedPlan) val broadcast = buildPlan.executeBroadcast[BuildSideRelation]() - VeloxBroadcastBuildSideRDD(sparkContext, broadcast) + val broadcastRDD = VeloxBroadcastBuildSideRDD(sparkContext, broadcast) + // FIXME: Do we have to make build side a RDD? + streamedRDD :+ broadcastRDD } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GlutenBroadcastNestedLoopJoinExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastNestedLoopJoinExecTransformer.scala similarity index 76% rename from backends-velox/src/main/scala/org/apache/gluten/execution/GlutenBroadcastNestedLoopJoinExecTransformer.scala rename to backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastNestedLoopJoinExecTransformer.scala index 3cde6b27b1fa..8517422698d9 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GlutenBroadcastNestedLoopJoinExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastNestedLoopJoinExecTransformer.scala @@ -16,13 +16,15 @@ */ package org.apache.gluten.execution +import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.optimizer.BuildSide import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.joins.BuildSideRelation +import org.apache.spark.sql.vectorized.ColumnarBatch -case class GlutenBroadcastNestedLoopJoinExecTransformer( +case class VeloxBroadcastNestedLoopJoinExecTransformer( left: SparkPlan, right: SparkPlan, buildSide: BuildSide, @@ -36,14 +38,17 @@ case class GlutenBroadcastNestedLoopJoinExecTransformer( condition ) { - override protected def createBroadcastBuildSideRDD(): BroadcastBuildSideRDD = { + override def columnarInputRDDs: Seq[RDD[ColumnarBatch]] = { + val streamedRDD = getColumnarInputRDDs(streamedPlan) val broadcast = buildPlan.executeBroadcast[BuildSideRelation]() - VeloxBroadcastBuildSideRDD(sparkContext, broadcast) + val broadcastRDD = VeloxBroadcastBuildSideRDD(sparkContext, broadcast) + // FIXME: Do we have to make build side a RDD? + streamedRDD :+ broadcastRDD } override protected def withNewChildrenInternal( newLeft: SparkPlan, - newRight: SparkPlan): GlutenBroadcastNestedLoopJoinExecTransformer = + newRight: SparkPlan): VeloxBroadcastNestedLoopJoinExecTransformer = copy(left = newLeft, right = newRight) } diff --git a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java index db54bd7832da..ba349a4f04b4 100644 --- a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java @@ -19,7 +19,10 @@ import org.apache.gluten.backendsapi.ListenerApi; import org.apache.gluten.backendsapi.velox.VeloxListenerApi; +import com.codahale.metrics.MetricRegistry; import org.apache.spark.SparkConf; +import org.apache.spark.api.plugin.PluginContext; +import org.apache.spark.resource.ResourceInformation; import org.apache.spark.util.TaskResources$; import org.apache.spark.util.sketch.BloomFilter; import org.apache.spark.util.sketch.IncompatibleMergeException; @@ -28,14 +31,52 @@ import org.junit.Test; import org.junit.function.ThrowingRunnable; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Map; public class VeloxBloomFilterTest { @BeforeClass public static void setup() { final ListenerApi api = new VeloxListenerApi(); - api.onDriverStart(new SparkConf()); + PluginContext pluginContext = + new PluginContext() { + @Override + public MetricRegistry metricRegistry() { + return null; + } + + @Override + public SparkConf conf() { + return new SparkConf(); + } + + @Override + public String executorID() { + return ""; + } + + @Override + public String hostname() { + return ""; + } + + @Override + public Map resources() { + return Collections.emptyMap(); + } + + @Override + public void send(Object message) throws IOException {} + + @Override + public Object ask(Object message) throws Exception { + return null; + } + }; + api.onDriverStart(null, pluginContext); } @Test diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml index 3934d535ef45..740de5928f26 100644 --- a/gluten-core/pom.xml +++ b/gluten-core/pom.xml @@ -24,10 +24,6 @@ gluten-ui ${project.version} - - com.github.ben-manes.caffeine - caffeine - org.apache.gluten ${sparkshim.artifactId} diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 6c3d62c1e207..7c601e48d4f6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -31,7 +31,6 @@ import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, import org.apache.spark.internal.Logging import org.apache.spark.listener.GlutenListenerFactory import org.apache.spark.network.util.JavaUtils -import org.apache.spark.rpc.{GlutenDriverEndpoint, GlutenExecutorEndpoint} import org.apache.spark.sql.SparkSessionExtensions import org.apache.spark.sql.execution.ui.GlutenEventUtils import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} @@ -73,8 +72,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { } // Initialize Backends API BackendsApiManager.initialize() - BackendsApiManager.getListenerApiInstance.onDriverStart(conf) - GlutenDriverEndpoint.glutenDriverEndpointRef = (new GlutenDriverEndpoint).self + BackendsApiManager.getListenerApiInstance.onDriverStart(sc, pluginContext) GlutenListenerFactory.addToSparkListenerBus(sc) ExpressionMappings.expressionExtensionTransformer = ExpressionUtil.extendedExpressionTransformer( @@ -257,7 +255,6 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { } private[gluten] class GlutenExecutorPlugin extends ExecutorPlugin { - private var executorEndpoint: GlutenExecutorEndpoint = _ private val taskListeners: Seq[TaskListener] = Array(TaskResources) /** Initialize the executor plugin. */ @@ -267,8 +264,7 @@ private[gluten] class GlutenExecutorPlugin extends ExecutorPlugin { // Initialize Backends API // TODO categorize the APIs by driver's or executor's BackendsApiManager.initialize() - BackendsApiManager.getListenerApiInstance.onExecutorStart(conf) - executorEndpoint = new GlutenExecutorEndpoint(ctx.executorID(), conf) + BackendsApiManager.getListenerApiInstance.onExecutorStart(ctx) } /** Clean up and terminate this plugin. For example: close the native engine. */ diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/Backend.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/Backend.scala index 6ad78e10536f..2c465ac61993 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/Backend.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/Backend.scala @@ -33,8 +33,6 @@ trait Backend { def listenerApi(): ListenerApi - def broadcastApi(): BroadcastApi - def settings(): BackendSettingsApi } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendsApiManager.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendsApiManager.scala index 1d9690d17842..f2c93d8c70fc 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendsApiManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendsApiManager.scala @@ -83,10 +83,6 @@ object BackendsApiManager { backend.metricsApi() } - def getBroadcastApiInstance: BroadcastApi = { - backend.broadcastApi() - } - def getSettings: BackendSettingsApi = { backend.settings } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BroadcastApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BroadcastApi.scala deleted file mode 100644 index 8b8b0d649a77..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BroadcastApi.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.backendsapi - -trait BroadcastApi { - - /** - * Should call by driver. Collect Broadcast Hash Table Ids. - * - * @param executionId - * execution id - * @param buildTableId - * build table id - */ - def collectExecutionBroadcastTableId(executionId: String, buildTableId: String): Unit = {} - - /** - * Should call by executor. On execution end. Clean executor broadcast build hashtable. - * - * @param executionId - * execution id - * @param broadcastTableIds - * broadcast table ids - */ - def cleanExecutionBroadcastTable( - executionId: String, - broadcastTableIds: java.util.Set[String]): Unit = {} -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/ListenerApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/ListenerApi.scala index aaba345fc4fd..bad169b72140 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/ListenerApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/ListenerApi.scala @@ -16,11 +16,12 @@ */ package org.apache.gluten.backendsapi -import org.apache.spark.SparkConf +import org.apache.spark.SparkContext +import org.apache.spark.api.plugin.PluginContext trait ListenerApi { - def onDriverStart(conf: SparkConf): Unit = {} + def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = {} def onDriverShutdown(): Unit = {} - def onExecutorStart(conf: SparkConf): Unit = {} + def onExecutorStart(pc: PluginContext): Unit = {} def onExecutorShutdown(): Unit = {} } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala index e41df0f2f240..05a639ac2dc7 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala @@ -70,4 +70,7 @@ trait TransformerApi { def getNativePlanString(substraitPlan: Array[Byte], details: Boolean): String def packPBMessage(message: Message): Any + + /** This method is only used for CH backend tests */ + def invalidateSQLExecutionResource(executionId: String): Unit = {} } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala index 2f666a811dab..092612ea7340 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala @@ -22,15 +22,13 @@ import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.utils.SubstraitUtil -import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} import org.apache.spark.sql.catalyst.plans.{InnerLike, JoinType, LeftOuter, RightOuter} import org.apache.spark.sql.catalyst.plans.physical.Partitioning -import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.joins.BaseJoinExec import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.vectorized.ColumnarBatch import io.substrait.proto.CrossRel @@ -66,20 +64,6 @@ abstract class BroadcastNestedLoopJoinExecTransformer( (right, left) } - override def columnarInputRDDs: Seq[RDD[ColumnarBatch]] = { - val streamedRDD = getColumnarInputRDDs(streamedPlan) - val broadcastRDD = { - val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - BackendsApiManager.getBroadcastApiInstance - .collectExecutionBroadcastTableId(executionId, buildTableId) - createBroadcastBuildSideRDD() - } - // FIXME: Do we have to make build side a RDD? - streamedRDD :+ broadcastRDD - } - - protected def createBroadcastBuildSideRDD(): BroadcastBuildSideRDD - @transient override lazy val metrics: Map[String, SQLMetric] = BackendsApiManager.getMetricsApiInstance.genNestedLoopJoinTransformerMetrics(sparkContext) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala index 0414c95aa918..6c707e5aa974 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical._ -import org.apache.spark.sql.execution.{ExpandOutputPartitioningShim, SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.{ExpandOutputPartitioningShim, SparkPlan} import org.apache.spark.sql.execution.joins.{BaseJoinExec, HashedRelationBroadcastMode, HashJoin} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types._ @@ -414,18 +414,4 @@ abstract class BroadcastHashJoinExecTransformerBase( override def genJoinParametersInternal(): (Int, Int, String) = { (1, if (isNullAwareAntiJoin) 1 else 0, buildHashTableId) } - - override def columnarInputRDDs: Seq[RDD[ColumnarBatch]] = { - val streamedRDD = getColumnarInputRDDs(streamedPlan) - val broadcastRDD = { - val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - BackendsApiManager.getBroadcastApiInstance - .collectExecutionBroadcastTableId(executionId, buildHashTableId) - createBroadcastBuildSideRDD() - } - // FIXME: Do we have to make build side a RDD? - streamedRDD :+ broadcastRDD - } - - protected def createBroadcastBuildSideRDD(): BroadcastBuildSideRDD } diff --git a/gluten-core/src/main/scala/org/apache/spark/listener/GlutenListenerFactory.scala b/gluten-core/src/main/scala/org/apache/spark/listener/GlutenListenerFactory.scala index 9413941fe598..721711af5fca 100644 --- a/gluten-core/src/main/scala/org/apache/spark/listener/GlutenListenerFactory.scala +++ b/gluten-core/src/main/scala/org/apache/spark/listener/GlutenListenerFactory.scala @@ -20,12 +20,9 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.softaffinity.scheduler.SoftAffinityListener import org.apache.spark.SparkContext -import org.apache.spark.rpc.GlutenDriverEndpoint object GlutenListenerFactory { def addToSparkListenerBus(sc: SparkContext): Unit = { - sc.listenerBus.addToStatusQueue( - new GlutenSQLAppStatusListener(GlutenDriverEndpoint.glutenDriverEndpointRef)) if ( sc.getConf.getBoolean( GlutenConfig.GLUTEN_SOFT_AFFINITY_ENABLED, diff --git a/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala b/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala index 35afc731bc2e..ab30cb14e180 100644 --- a/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala +++ b/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala @@ -20,10 +20,10 @@ package org.apache.spark.sql * Why we need a GlutenQueryTest when we already have QueryTest? * 1. We need to modify the way org.apache.spark.sql.CHQueryTest#compare compares double */ +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SPARK_VERSION_SHORT -import org.apache.spark.rpc.GlutenDriverEndpoint import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ @@ -339,7 +339,7 @@ object GlutenQueryTest extends Assertions { SQLExecution.withExecutionId(df.sparkSession, executionId) { df.rdd.count() // Also attempt to deserialize as an RDD [SPARK-15791] } - GlutenDriverEndpoint.invalidateResourceRelation(executionId) + BackendsApiManager.getTransformerApiInstance.invalidateSQLExecutionResource(executionId) } val sparkAnswer = From 4c9131a51f11e0f6f4796d58ca75d0ee08a01c09 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 31 May 2024 09:29:04 +0800 Subject: [PATCH 184/402] [VL] Gluten-it: Optimize Maven dependency list --- tools/gluten-it/common/pom.xml | 11 ++++ tools/gluten-it/package/pom.xml | 11 ++++ tools/gluten-it/pom.xml | 95 +++++++-------------------------- 3 files changed, 41 insertions(+), 76 deletions(-) diff --git a/tools/gluten-it/common/pom.xml b/tools/gluten-it/common/pom.xml index 4cd4684724ad..de0d7c2e6c8c 100644 --- a/tools/gluten-it/common/pom.xml +++ b/tools/gluten-it/common/pom.xml @@ -57,6 +57,17 @@ provided test-jar + + org.apache.spark + spark-sql_${scala.binary.version} + provided + + + org.apache.spark + spark-sql_${scala.binary.version} + provided + test-jar + io.trino.tpcds diff --git a/tools/gluten-it/package/pom.xml b/tools/gluten-it/package/pom.xml index 1f86ee723240..70a59cac05ea 100644 --- a/tools/gluten-it/package/pom.xml +++ b/tools/gluten-it/package/pom.xml @@ -69,6 +69,17 @@ runtime test-jar + + org.apache.spark + spark-sql_${scala.binary.version} + runtime + + + org.apache.spark + spark-sql_${scala.binary.version} + runtime + test-jar + diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index c0e2fc321643..ccb59ade8aa5 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -87,6 +87,25 @@ provided test-jar + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + + + com.google.protobuf + protobuf-java + + + provided + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + test-jar + provided + @@ -117,100 +136,24 @@ 3.2.2 - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - - - com.google.protobuf - protobuf-java - - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - test-jar - - spark-3.3 3.3.1 - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - - - com.google.protobuf - protobuf-java - - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - test-jar - - spark-3.4 3.4.2 - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - - - com.google.protobuf - protobuf-java - - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - test-jar - - spark-3.5 3.5.1 - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - - - com.google.protobuf - protobuf-java - - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - test-jar - - celeborn-0.4 From c87029f521489c805bbf718c6e02a0ecc7d4266b Mon Sep 17 00:00:00 2001 From: Shuai li Date: Fri, 31 May 2024 09:45:48 +0800 Subject: [PATCH 185/402] [GLUTEN-5921][CH] Function trim of trim_character support value from column (#5922) [CH] Function trim of trim_character support value from column --- .../GlutenFunctionValidateSuite.scala | 18 ++++- .../Functions/SparkFunctionTrim.cpp | 76 +++++++++++++++---- 2 files changed, 80 insertions(+), 14 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index 5a1ca679986f..cfe8ea95abcd 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -708,6 +708,23 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS } + test("GLUTEN-5821: trim_character support value from column.") { + withTable("trim") { + sql("create table trim(a String, b String) using parquet") + sql(""" + |insert into trim values ('aba', 'a'),('bba', 'b'),('abcdef', 'abcd') + |""".stripMargin) + + val sql_str = + s"""select + | trim(both b from a) + | from trim + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + test("GLUTEN-5897: fix regexp_extract with bracket") { withTable("regexp_extract_bracket") { sql("create table regexp_extract_bracket(a String) using parquet") @@ -727,5 +744,4 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS runQueryAndCompare(sql_str) { _ => } } } - } diff --git a/cpp-ch/local-engine/Functions/SparkFunctionTrim.cpp b/cpp-ch/local-engine/Functions/SparkFunctionTrim.cpp index d8f6be1bfc32..88ed3f635672 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionTrim.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionTrim.cpp @@ -77,8 +77,6 @@ namespace bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.size() != 2) @@ -112,19 +110,34 @@ namespace if (!src_str_col) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be String", getName()); - const ColumnConst * trim_str_col = checkAndGetColumnConst(arguments[1].column.get()); - if (!trim_str_col) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be Const String", getName()); - - String trim_str = trim_str_col->getValue(); - if (trim_str.empty()) - return src_str_col->cloneResized(input_rows_count); - auto res_col = ColumnString::create(); - res_col->reserve(input_rows_count); + if (const auto * trim_const_str_col = checkAndGetColumnConst(arguments[1].column.get())) + { + String trim_str = trim_const_str_col->getValue(); + if (trim_str.empty()) + return src_str_col->cloneResized(input_rows_count); + + auto res_col = ColumnString::create(); + res_col->reserve(input_rows_count); + executeVector(src_str_col->getChars(), src_str_col->getOffsets(), res_col->getChars(), res_col->getOffsets(), trim_str); + return std::move(res_col); + } + else if (const auto * trim_str_col = checkAndGetColumn(arguments[1].column.get())) + { + auto res_col = ColumnString::create(); + res_col->reserve(input_rows_count); + + executeVector( + src_str_col->getChars(), + src_str_col->getOffsets(), + res_col->getChars(), + res_col->getOffsets(), + trim_str_col->getChars(), + trim_str_col->getOffsets()); + return std::move(res_col); + } - executeVector(src_str_col->getChars(), src_str_col->getOffsets(), res_col->getChars(), res_col->getOffsets(), trim_str); - return std::move(res_col); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be String or Const String", getName()); } private: @@ -159,6 +172,43 @@ namespace } } + void executeVector( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets, + const ColumnString::Chars & trim_data, + const ColumnString::Offsets & trim_offsets) const + { + res_data.reserve_exact(data.size()); + + size_t rows = offsets.size(); + res_offsets.resize_exact(rows); + + size_t prev_offset = 0; + size_t prev_trim_str_offset = 0; + size_t res_offset = 0; + + const UInt8 * start; + size_t length; + + for (size_t i = 0; i < rows; ++i) + { + std::unordered_set trim_set( + &trim_data[prev_trim_str_offset], &trim_data[prev_trim_str_offset] + trim_offsets[i] - prev_trim_str_offset - 1); + + trim(reinterpret_cast(&data[prev_offset]), offsets[i] - prev_offset - 1, start, length, trim_set); + res_data.resize_exact(res_data.size() + length + 1); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], start, length); + res_offset += length + 1; + res_data[res_offset - 1] = '\0'; + + res_offsets[i] = res_offset; + prev_offset = offsets[i]; + prev_trim_str_offset = trim_offsets[i]; + } + } + void trim(const UInt8 * data, size_t size, const UInt8 *& res_data, size_t & res_size, const std::unordered_set & trim_set) const { From f48b9fa09f6421f861a22d8027d2cb81767f4e5c Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Fri, 31 May 2024 11:12:28 +0800 Subject: [PATCH 186/402] [CORE] Use the smaller table to build hashmap in shuffled hash join (#5750) --- .../backendsapi/velox/VeloxBackend.scala | 20 +- .../gluten/extension/StrategyOverrides.scala | 12 +- .../columnar/OffloadSingleNode.scala | 43 +- .../columnar/TransformHintRule.scala | 31 +- .../gluten/planner/cost/GlutenCostModel.scala | 6 - .../execution/joins/HashJoin.scala.deprecated | 774 ------------------ .../execution/joins/HashJoin.scala.deprecated | 774 ------------------ .../execution/joins/HashJoin.scala.deprecated | 774 ------------------ .../execution/joins/HashJoin.scala.deprecated | 774 ------------------ 9 files changed, 41 insertions(+), 3167 deletions(-) delete mode 100644 shims/spark32/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated delete mode 100644 shims/spark33/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated delete mode 100644 shims/spark34/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated delete mode 100644 shims/spark35/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index c8dbfb29e485..f06929fff620 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -29,7 +29,7 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat.{DwrfReadFo import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, Expression, Lag, Lead, Literal, MakeYMInterval, NamedExpression, NthValue, NTile, PercentRank, Rand, RangeFrame, Rank, RowNumber, SortOrder, SparkPartitionID, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, ApproximatePercentile, Count, Sum} -import org.apache.spark.sql.catalyst.plans.JoinType +import org.apache.spark.sql.catalyst.plans.{JoinType, LeftOuter, RightOuter} import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.spark.sql.execution.aggregate.HashAggregateExec @@ -375,13 +375,10 @@ object VeloxBackendSettings extends BackendSettingsApi { } else { t match { // OPPRO-266: For Velox backend, build right and left are both supported for - // LeftOuter and LeftSemi. - // FIXME Hongze 22/12/06 - // HashJoin.scala in shim was not always loaded by class loader. - // The file should be removed and we temporarily disable the improvement - // introduced by OPPRO-266 by commenting out the following prerequisite - // condition. -// case LeftOuter | LeftSemi => true + // LeftOuter. + // TODO: Support LeftSemi after resolve issue + // https://github.com/facebookincubator/velox/issues/9980 + case LeftOuter => true case _ => false } } @@ -393,12 +390,7 @@ object VeloxBackendSettings extends BackendSettingsApi { } else { t match { // OPPRO-266: For Velox backend, build right and left are both supported for RightOuter. - // FIXME Hongze 22/12/06 - // HashJoin.scala in shim was not always loaded by class loader. - // The file should be removed and we temporarily disable the improvement - // introduced by OPPRO-266 by commenting out the following prerequisite - // condition. -// case RightOuter => true + case RightOuter => true case _ => false } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala index d016eacccf11..f2f786259393 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala @@ -111,7 +111,9 @@ case class JoinSelectionOverrides(session: SparkSession) // it don't use this side as the build side (!leftHintMergeEnabled, !rightHintMergeEnabled) } else { - (canBuildShuffledHashJoinLeft(joinType), canBuildShuffledHashJoinRight(joinType)) + ( + BackendsApiManager.getSettings.supportHashBuildJoinTypeOnLeft(joinType), + BackendsApiManager.getSettings.supportHashBuildJoinTypeOnRight(joinType)) } } else { (canBuildShuffledHashJoinLeft(joinType), canBuildShuffledHashJoinRight(joinType)) @@ -147,14 +149,6 @@ case class JoinSelectionOverrides(session: SparkSession) } } - override def canBuildShuffledHashJoinLeft(joinType: JoinType): Boolean = { - BackendsApiManager.getSettings.supportHashBuildJoinTypeOnLeft(joinType) - } - - override def canBuildShuffledHashJoinRight(joinType: JoinType): Boolean = { - BackendsApiManager.getSettings.supportHashBuildJoinTypeOnRight(joinType) - } - def existsMultiJoins(plan: LogicalPlan, count: Int = 0): Boolean = { plan match { case plan: Join => diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index e0aa0c26bb35..b82254072033 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -28,8 +28,6 @@ import org.apache.gluten.utils.{LogLevelUtil, PlanUtil} import org.apache.spark.api.python.EvalPythonExecTransformer import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans.{LeftOuter, LeftSemi, RightOuter} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} import org.apache.spark.sql.execution.datasources.WriteFilesExec @@ -126,34 +124,11 @@ case class OffloadExchange() extends OffloadSingleNode with LogLevelUtil { // Join transformation. case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { - import OffloadJoin._ override def offload(plan: SparkPlan): SparkPlan = { if (TransformHints.isNotTransformable(plan)) { logDebug(s"Columnar Processing for ${plan.getClass} is under row guard.") - plan match { - case shj: ShuffledHashJoinExec => - if (BackendsApiManager.getSettings.recreateJoinExecOnFallback()) { - // Since https://github.com/apache/incubator-gluten/pull/408 - // Because we manually removed the build side limitation for LeftOuter, LeftSemi and - // RightOuter, need to change the build side back if this join fallback into vanilla - // Spark for execution. - return ShuffledHashJoinExec( - shj.leftKeys, - shj.rightKeys, - shj.joinType, - getSparkSupportedBuildSide(shj), - shj.condition, - shj.left, - shj.right, - shj.isSkewJoin - ) - } else { - return shj - } - case p => - return p - } + return plan } plan match { case plan: ShuffledHashJoinExec => @@ -165,7 +140,7 @@ case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { plan.leftKeys, plan.rightKeys, plan.joinType, - plan.buildSide, + TransformHints.getShuffleHashJoinBuildSide(plan), plan.condition, left, right, @@ -217,20 +192,6 @@ case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { } -object OffloadJoin { - private def getSparkSupportedBuildSide(plan: ShuffledHashJoinExec): BuildSide = { - plan.joinType match { - case LeftOuter | LeftSemi => BuildRight - case RightOuter => BuildLeft - case _ => plan.buildSide - } - } - - def isLegal(plan: ShuffledHashJoinExec): Boolean = { - plan.buildSide == getSparkSupportedBuildSide(plan) - } -} - // Filter transformation. case class OffloadFilter() extends OffloadSingleNode with LogLevelUtil { import OffloadOthers._ diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index 7ce9ffc52d67..1da0c0db8a6d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -29,6 +29,8 @@ import org.apache.spark.api.python.EvalPythonExecTransformer import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} +import org.apache.spark.sql.catalyst.plans.logical.Join import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.spark.sql.execution._ @@ -158,6 +160,33 @@ object TransformHints { tag(plan, newTag) } } + + def getShuffleHashJoinBuildSide(shj: ShuffledHashJoinExec): BuildSide = { + if (BackendsApiManager.getSettings.utilizeShuffledHashJoinHint()) { + shj.buildSide + } else { + val leftBuildable = BackendsApiManager.getSettings + .supportHashBuildJoinTypeOnLeft(shj.joinType) + val rightBuildable = BackendsApiManager.getSettings + .supportHashBuildJoinTypeOnRight(shj.joinType) + + if (!leftBuildable) { + BuildRight + } else if (!rightBuildable) { + BuildLeft + } else { + shj.logicalLink match { + case Some(join: Join) => + val leftSize = join.left.stats.sizeInBytes + val rightSize = join.right.stats.sizeInBytes + if (rightSize <= leftSize) BuildRight else BuildLeft + // Only the ShuffledHashJoinExec generated directly in some spark tests is not link + // logical plan, such as OuterJoinSuite. + case _ => shj.buildSide + } + } + } + } } case class FallbackOnANSIMode(session: SparkSession) extends Rule[SparkPlan] { @@ -392,7 +421,7 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { plan.leftKeys, plan.rightKeys, plan.joinType, - plan.buildSide, + TransformHints.getShuffleHashJoinBuildSide(plan), plan.condition, plan.left, plan.right, diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala index c45314a9f58f..ab0deab1939d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/cost/GlutenCostModel.scala @@ -17,7 +17,6 @@ package org.apache.gluten.planner.cost import org.apache.gluten.GlutenConfig -import org.apache.gluten.extension.columnar.OffloadJoin import org.apache.gluten.extension.columnar.enumerated.RemoveFilter import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike} import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec @@ -26,7 +25,6 @@ import org.apache.gluten.utils.PlanUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan} -import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec import org.apache.spark.sql.utils.ReflectionUtil object GlutenCostModel extends Logging { @@ -75,10 +73,6 @@ object GlutenCostModel extends Logging { // A very rough estimation as of now. private def selfLongCostOf(node: SparkPlan): Long = { node match { - case p: ShuffledHashJoinExec if !OffloadJoin.isLegal(p) => - // To exclude the rewritten intermediate plan that is not executable - // by vanilla Spark and was generated by strategy "JoinSelectionOverrides" - infLongCost case _: RemoveFilter.NoopFilter => // To make planner choose the tree that has applied rule PushFilterToScan. 0L diff --git a/shims/spark32/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated b/shims/spark32/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated deleted file mode 100644 index 24bf246b325c..000000000000 --- a/shims/spark32/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated +++ /dev/null @@ -1,774 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} -import org.apache.spark.sql.catalyst.analysis.CastSupport -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences -import org.apache.spark.sql.catalyst.expressions.codegen._ -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.physical.Partitioning -import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, RowIterator} -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.types.{BooleanType, IntegralType, LongType} - -/** - * @param relationTerm variable name for HashedRelation - * @param keyIsUnique indicate whether keys of HashedRelation known to be unique in code-gen time - * @param isEmpty indicate whether it known to be EmptyHashedRelation in code-gen time - */ -private[joins] case class HashedRelationInfo( - relationTerm: String, - keyIsUnique: Boolean, - isEmpty: Boolean) - -trait HashJoin extends JoinCodegenSupport { - def buildSide: BuildSide - - override def simpleStringWithNodeId(): String = { - val opId = ExplainUtils.getOpId(this) - s"$nodeName $joinType ${buildSide} ($opId)".trim - } - - override def output: Seq[Attribute] = { - joinType match { - case _: InnerLike => - left.output ++ right.output - case LeftOuter => - left.output ++ right.output.map(_.withNullability(true)) - case RightOuter => - left.output.map(_.withNullability(true)) ++ right.output - case j: ExistenceJoin => - left.output :+ j.exists - case LeftExistence(_) => - left.output - case x => - throw new IllegalArgumentException(s"HashJoin should not take $x as the JoinType") - } - } - - override def outputPartitioning: Partitioning = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - /** - * Handle the special cases for LeftOuter/LeftSemi with BuildLeft and RightOuter with BuildRight. - */ - override def outputOrdering: Seq[SortOrder] = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputOrdering - case LeftOuter => left.outputOrdering - case LeftSemi => left.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputOrdering - case RightOuter => right.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - protected lazy val (buildPlan, streamedPlan) = buildSide match { - case BuildLeft => (left, right) - case BuildRight => (right, left) - } - - protected lazy val (buildKeys, streamedKeys) = { - require(leftKeys.length == rightKeys.length && - leftKeys.map(_.dataType) - .zip(rightKeys.map(_.dataType)) - .forall(types => types._1.sameType(types._2)), - "Join keys from two sides should have same length and types") - buildSide match { - case BuildLeft => (leftKeys, rightKeys) - case BuildRight => (rightKeys, leftKeys) - } - } - - @transient protected lazy val (buildOutput, streamedOutput) = { - buildSide match { - case BuildLeft => (left.output, right.output) - case BuildRight => (right.output, left.output) - } - } - - @transient protected lazy val buildBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(buildKeys), buildOutput) - - @transient protected lazy val streamedBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(streamedKeys), streamedOutput) - - protected def buildSideKeyGenerator(): Projection = - UnsafeProjection.create(buildBoundKeys) - - protected def streamSideKeyGenerator(): UnsafeProjection = - UnsafeProjection.create(streamedBoundKeys) - - @transient protected[this] lazy val boundCondition = if (condition.isDefined) { - if (joinType == FullOuter && buildSide == BuildLeft) { - // Put join left side before right side. This is to be consistent with - // `ShuffledHashJoinExec.fullOuterJoin`. - Predicate.create(condition.get, buildPlan.output ++ streamedPlan.output).eval _ - } else { - Predicate.create(condition.get, streamedPlan.output ++ buildPlan.output).eval _ - } - } else { - (r: InternalRow) => true - } - - protected def createResultProjection(): (InternalRow) => InternalRow = joinType match { - case LeftExistence(_) => - UnsafeProjection.create(output, output) - case _ => - // Always put the stream side on left to simplify implementation - // both of left and right side could be null - UnsafeProjection.create( - output, (streamedPlan.output ++ buildPlan.output).map(_.withNullability(true))) - } - - private def innerJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinRow = new JoinedRow - val joinKeys = streamSideKeyGenerator() - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matched = hashedRelation.getValue(joinKeys(srow)) - if (matched != null) { - Some(joinRow.withRight(matched)).filter(boundCondition) - } else { - None - } - } - } else { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matches = hashedRelation.get(joinKeys(srow)) - if (matches != null) { - matches.map(joinRow.withRight).filter(boundCondition) - } else { - Seq.empty - } - } - } - } - - private def outerJoin( - streamedIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinedRow = new JoinedRow() - val keyGenerator = streamSideKeyGenerator() - val nullRow = new GenericInternalRow(buildPlan.output.length) - - if (hashedRelation.keyIsUnique) { - streamedIter.map { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val matched = hashedRelation.getValue(rowKey) - if (matched != null && boundCondition(joinedRow.withRight(matched))) { - joinedRow - } else { - joinedRow.withRight(nullRow) - } - } - } else { - streamedIter.flatMap { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val buildIter = hashedRelation.get(rowKey) - new RowIterator { - private var found = false - override def advanceNext(): Boolean = { - while (buildIter != null && buildIter.hasNext) { - val nextBuildRow = buildIter.next() - if (boundCondition(joinedRow.withRight(nextBuildRow))) { - found = true - return true - } - } - if (!found) { - joinedRow.withRight(nullRow) - found = true - return true - } - false - } - override def getRow: InternalRow = joinedRow - }.toScala - } - } - } - - private def semiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - } - } - } - - private def existenceJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val result = new GenericInternalRow(Array[Any](null)) - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.map { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - val exists = !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } else { - streamIter.map { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - val exists = !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } - } - - private def antiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - // If the right side is empty, AntiJoin simply returns the left side. - if (hashedRelation == EmptyHashedRelation) { - return streamIter - } - - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - key.anyNull || matched == null || - (condition.isDefined && !boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - key.anyNull || buildIter == null || (condition.isDefined && !buildIter.exists { - row => boundCondition(joinedRow(current, row)) - }) - } - } - } - - protected def join( - streamedIter: Iterator[InternalRow], - hashed: HashedRelation, - numOutputRows: SQLMetric): Iterator[InternalRow] = { - - val joinedIter = joinType match { - case _: InnerLike => - innerJoin(streamedIter, hashed) - case LeftOuter | RightOuter => - outerJoin(streamedIter, hashed) - case LeftSemi => - semiJoin(streamedIter, hashed) - case LeftAnti => - antiJoin(streamedIter, hashed) - case _: ExistenceJoin => - existenceJoin(streamedIter, hashed) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - - val resultProj = createResultProjection - joinedIter.map { r => - numOutputRows += 1 - resultProj(r) - } - } - - override def doProduce(ctx: CodegenContext): String = { - streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this) - } - - override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - joinType match { - case _: InnerLike => codegenInner(ctx, input) - case LeftOuter | RightOuter => codegenOuter(ctx, input) - case LeftSemi => codegenSemi(ctx, input) - case LeftAnti => codegenAnti(ctx, input) - case _: ExistenceJoin => codegenExistence(ctx, input) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - } - - /** - * Returns the code for generating join key for stream side, and expression of whether the key - * has any null in it or not. - */ - protected def genStreamSideJoinKey( - ctx: CodegenContext, - input: Seq[ExprCode]): (ExprCode, String) = { - ctx.currentVars = input - if (streamedBoundKeys.length == 1 && streamedBoundKeys.head.dataType == LongType) { - // generate the join key as Long - val ev = streamedBoundKeys.head.genCode(ctx) - (ev, ev.isNull) - } else { - // generate the join key as UnsafeRow - val ev = GenerateUnsafeProjection.createCode(ctx, streamedBoundKeys) - (ev, s"${ev.value}.anyNull()") - } - } - - /** - * Generates the code for Inner join. - */ - protected def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, buildVars) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash inner join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? - | null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | while ($matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left or right outer join. - */ - protected def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val matched = ctx.freshName("matched") - val buildVars = genBuildSideVars(ctx, matched, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - // filter the output via condition - val conditionPassed = ctx.freshName("conditionPassed") - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |boolean $conditionPassed = true; - |${eval.trim} - |if ($matched != null) { - | ${ev.code} - | $conditionPassed = !${ev.isNull} && ${ev.value}; - |} - """.stripMargin - } else { - s"final boolean $conditionPassed = true;" - } - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |${checkCondition.trim} - |if (!$conditionPassed) { - | $matched = null; - | // reset the variables those are already evaluated. - | ${buildVars.filter(_.code.isEmpty).map(v => s"${v.isNull} = true;").mkString("\n")} - |} - |$numOutput.add(1); - |${consume(ctx, resultVars)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $found = false; - |// the last iteration of this loop is to emit an empty row if there is no matched rows. - |while ($matches != null && $matches.hasNext() || !$found) { - | UnsafeRow $matched = $matches != null && $matches.hasNext() ? - | (UnsafeRow) $matches.next() : null; - | ${checkCondition.trim} - | if ($conditionPassed) { - | $found = true; - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left semi join. - */ - protected def codegenSemi(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash semi join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | boolean $found = false; - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | if ($found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for anti join. - */ - protected def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val numOutput = metricTerm(ctx, "numOutputRows") - if (isEmptyHashedRelation) { - return s""" - |// If HashedRelation is empty, hash anti join simply returns the stream side. - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } - - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - - if (keyIsUnique) { - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); - | if ($matches != null) { - | // Evaluate the condition. - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } - } - - /** - * Generates the code for existence join. - */ - protected def codegenExistence(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val numOutput = metricTerm(ctx, "numOutputRows") - val existsVar = ctx.freshName("exists") - - val matched = ctx.freshName("matched") - val buildVars = genBuildSideVars(ctx, matched, buildPlan) - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - // filter the output via condition - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |$eval - |${ev.code} - |$existsVar = !${ev.isNull} && ${ev.value}; - """.stripMargin - } else { - s"$existsVar = true;" - } - - val resultVar = input ++ Seq(ExprCode.forNonNullValue( - JavaCode.variable(existsVar, BooleanType))) - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |boolean $existsVar = false; - |if ($matched != null) { - | $checkCondition - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $existsVar = false; - |if ($matches != null) { - | while (!$existsVar && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition - | } - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } - } - - protected def prepareRelation(ctx: CodegenContext): HashedRelationInfo -} - -object HashJoin extends CastSupport with SQLConfHelper { - - private def canRewriteAsLongType(keys: Seq[Expression]): Boolean = { - // TODO: support BooleanType, DateType and TimestampType - keys.forall(_.dataType.isInstanceOf[IntegralType]) && - keys.map(_.dataType.defaultSize).sum <= 8 - } - - /** - * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long. - * - * If not, returns the original expressions. - */ - def rewriteKeyExpr(keys: Seq[Expression]): Seq[Expression] = { - assert(keys.nonEmpty) - if (!canRewriteAsLongType(keys)) { - return keys - } - - var keyExpr: Expression = if (keys.head.dataType != LongType) { - cast(keys.head, LongType) - } else { - keys.head - } - keys.tail.foreach { e => - val bits = e.dataType.defaultSize * 8 - keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)), - BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1))) - } - keyExpr :: Nil - } - - /** - * Extract a given key which was previously packed in a long value using its index to - * determine the number of bits to shift - */ - def extractKeyExprAt(keys: Seq[Expression], index: Int): Expression = { - assert(canRewriteAsLongType(keys)) - // jump over keys that have a higher index value than the required key - if (keys.size == 1) { - assert(index == 0) - Cast( - child = BoundReference(0, LongType, nullable = false), - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } else { - val shiftedBits = - keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum - val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1 - // build the schema for unpacking the required key - val castChild = BitwiseAnd( - ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)), - Literal(mask)) - Cast( - child = castChild, - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } - } -} diff --git a/shims/spark33/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated b/shims/spark33/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated deleted file mode 100644 index 3fc823730614..000000000000 --- a/shims/spark33/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated +++ /dev/null @@ -1,774 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} -import org.apache.spark.sql.catalyst.analysis.CastSupport -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences -import org.apache.spark.sql.catalyst.expressions.codegen._ -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.physical.Partitioning -import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, RowIterator} -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.types.{BooleanType, IntegralType, LongType} - -/** - * @param relationTerm variable name for HashedRelation - * @param keyIsUnique indicate whether keys of HashedRelation known to be unique in code-gen time - * @param isEmpty indicate whether it known to be EmptyHashedRelation in code-gen time - */ -private[joins] case class HashedRelationInfo( - relationTerm: String, - keyIsUnique: Boolean, - isEmpty: Boolean) - -trait HashJoin extends JoinCodegenSupport { - def buildSide: BuildSide - - override def simpleStringWithNodeId(): String = { - val opId = ExplainUtils.getOpId(this) - s"$nodeName $joinType ${buildSide} ($opId)".trim - } - - override def output: Seq[Attribute] = { - joinType match { - case _: InnerLike => - left.output ++ right.output - case LeftOuter => - left.output ++ right.output.map(_.withNullability(true)) - case RightOuter => - left.output.map(_.withNullability(true)) ++ right.output - case j: ExistenceJoin => - left.output :+ j.exists - case LeftExistence(_) => - left.output - case x => - throw new IllegalArgumentException(s"HashJoin should not take $x as the JoinType") - } - } - - override def outputPartitioning: Partitioning = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - /** - * Handle the special cases for LeftOuter/LeftSemi with BuildLeft and RightOuter with BuildRight. - */ - override def outputOrdering: Seq[SortOrder] = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputOrdering - case LeftOuter => left.outputOrdering - case LeftSemi => left.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputOrdering - case RightOuter => right.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - protected lazy val (buildPlan, streamedPlan) = buildSide match { - case BuildLeft => (left, right) - case BuildRight => (right, left) - } - - protected lazy val (buildKeys, streamedKeys) = { - require(leftKeys.length == rightKeys.length && - leftKeys.map(_.dataType) - .zip(rightKeys.map(_.dataType)) - .forall(types => types._1.sameType(types._2)), - "Join keys from two sides should have same length and types") - buildSide match { - case BuildLeft => (leftKeys, rightKeys) - case BuildRight => (rightKeys, leftKeys) - } - } - - @transient protected lazy val (buildOutput, streamedOutput) = { - buildSide match { - case BuildLeft => (left.output, right.output) - case BuildRight => (right.output, left.output) - } - } - - @transient protected lazy val buildBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(buildKeys), buildOutput) - - @transient protected lazy val streamedBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(streamedKeys), streamedOutput) - - protected def buildSideKeyGenerator(): Projection = - UnsafeProjection.create(buildBoundKeys) - - protected def streamSideKeyGenerator(): UnsafeProjection = - UnsafeProjection.create(streamedBoundKeys) - - @transient protected[this] lazy val boundCondition = if (condition.isDefined) { - if (joinType == FullOuter && buildSide == BuildLeft) { - // Put join left side before right side. This is to be consistent with - // `ShuffledHashJoinExec.fullOuterJoin`. - Predicate.create(condition.get, buildPlan.output ++ streamedPlan.output).eval _ - } else { - Predicate.create(condition.get, streamedPlan.output ++ buildPlan.output).eval _ - } - } else { - (r: InternalRow) => true - } - - protected def createResultProjection(): (InternalRow) => InternalRow = joinType match { - case LeftExistence(_) => - UnsafeProjection.create(output, output) - case _ => - // Always put the stream side on left to simplify implementation - // both of left and right side could be null - UnsafeProjection.create( - output, (streamedPlan.output ++ buildPlan.output).map(_.withNullability(true))) - } - - private def innerJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinRow = new JoinedRow - val joinKeys = streamSideKeyGenerator() - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matched = hashedRelation.getValue(joinKeys(srow)) - if (matched != null) { - Some(joinRow.withRight(matched)).filter(boundCondition) - } else { - None - } - } - } else { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matches = hashedRelation.get(joinKeys(srow)) - if (matches != null) { - matches.map(joinRow.withRight).filter(boundCondition) - } else { - Seq.empty - } - } - } - } - - private def outerJoin( - streamedIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinedRow = new JoinedRow() - val keyGenerator = streamSideKeyGenerator() - val nullRow = new GenericInternalRow(buildPlan.output.length) - - if (hashedRelation.keyIsUnique) { - streamedIter.map { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val matched = hashedRelation.getValue(rowKey) - if (matched != null && boundCondition(joinedRow.withRight(matched))) { - joinedRow - } else { - joinedRow.withRight(nullRow) - } - } - } else { - streamedIter.flatMap { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val buildIter = hashedRelation.get(rowKey) - new RowIterator { - private var found = false - override def advanceNext(): Boolean = { - while (buildIter != null && buildIter.hasNext) { - val nextBuildRow = buildIter.next() - if (boundCondition(joinedRow.withRight(nextBuildRow))) { - found = true - return true - } - } - if (!found) { - joinedRow.withRight(nullRow) - found = true - return true - } - false - } - override def getRow: InternalRow = joinedRow - }.toScala - } - } - } - - private def semiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - } - } - } - - private def existenceJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val result = new GenericInternalRow(Array[Any](null)) - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.map { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - val exists = !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } else { - streamIter.map { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - val exists = !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } - } - - private def antiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - // If the right side is empty, AntiJoin simply returns the left side. - if (hashedRelation == EmptyHashedRelation) { - return streamIter - } - - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - key.anyNull || matched == null || - (condition.isDefined && !boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - key.anyNull || buildIter == null || (condition.isDefined && !buildIter.exists { - row => boundCondition(joinedRow(current, row)) - }) - } - } - } - - protected def join( - streamedIter: Iterator[InternalRow], - hashed: HashedRelation, - numOutputRows: SQLMetric): Iterator[InternalRow] = { - - val joinedIter = joinType match { - case _: InnerLike => - innerJoin(streamedIter, hashed) - case LeftOuter | RightOuter => - outerJoin(streamedIter, hashed) - case LeftSemi => - semiJoin(streamedIter, hashed) - case LeftAnti => - antiJoin(streamedIter, hashed) - case _: ExistenceJoin => - existenceJoin(streamedIter, hashed) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - - val resultProj = createResultProjection - joinedIter.map { r => - numOutputRows += 1 - resultProj(r) - } - } - - override def doProduce(ctx: CodegenContext): String = { - streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this) - } - - override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - joinType match { - case _: InnerLike => codegenInner(ctx, input) - case LeftOuter | RightOuter => codegenOuter(ctx, input) - case LeftSemi => codegenSemi(ctx, input) - case LeftAnti => codegenAnti(ctx, input) - case _: ExistenceJoin => codegenExistence(ctx, input) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - } - - /** - * Returns the code for generating join key for stream side, and expression of whether the key - * has any null in it or not. - */ - protected def genStreamSideJoinKey( - ctx: CodegenContext, - input: Seq[ExprCode]): (ExprCode, String) = { - ctx.currentVars = input - if (streamedBoundKeys.length == 1 && streamedBoundKeys.head.dataType == LongType) { - // generate the join key as Long - val ev = streamedBoundKeys.head.genCode(ctx) - (ev, ev.isNull) - } else { - // generate the join key as UnsafeRow - val ev = GenerateUnsafeProjection.createCode(ctx, streamedBoundKeys) - (ev, s"${ev.value}.anyNull()") - } - } - - /** - * Generates the code for Inner join. - */ - protected def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, buildVars) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash inner join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? - | null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | while ($matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left or right outer join. - */ - protected def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val matched = ctx.freshName("matched") - val buildVars = genOneSideJoinVars(ctx, matched, buildPlan, setDefaultValue = true) - val numOutput = metricTerm(ctx, "numOutputRows") - - // filter the output via condition - val conditionPassed = ctx.freshName("conditionPassed") - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |boolean $conditionPassed = true; - |${eval.trim} - |if ($matched != null) { - | ${ev.code} - | $conditionPassed = !${ev.isNull} && ${ev.value}; - |} - """.stripMargin - } else { - s"final boolean $conditionPassed = true;" - } - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |${checkCondition.trim} - |if (!$conditionPassed) { - | $matched = null; - | // reset the variables those are already evaluated. - | ${buildVars.filter(_.code.isEmpty).map(v => s"${v.isNull} = true;").mkString("\n")} - |} - |$numOutput.add(1); - |${consume(ctx, resultVars)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $found = false; - |// the last iteration of this loop is to emit an empty row if there is no matched rows. - |while ($matches != null && $matches.hasNext() || !$found) { - | UnsafeRow $matched = $matches != null && $matches.hasNext() ? - | (UnsafeRow) $matches.next() : null; - | ${checkCondition.trim} - | if ($conditionPassed) { - | $found = true; - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left semi join. - */ - protected def codegenSemi(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash semi join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | boolean $found = false; - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | if ($found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for anti join. - */ - protected def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val numOutput = metricTerm(ctx, "numOutputRows") - if (isEmptyHashedRelation) { - return s""" - |// If HashedRelation is empty, hash anti join simply returns the stream side. - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } - - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - - if (keyIsUnique) { - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); - | if ($matches != null) { - | // Evaluate the condition. - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } - } - - /** - * Generates the code for existence join. - */ - protected def codegenExistence(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val numOutput = metricTerm(ctx, "numOutputRows") - val existsVar = ctx.freshName("exists") - - val matched = ctx.freshName("matched") - val buildVars = genOneSideJoinVars(ctx, matched, buildPlan, setDefaultValue = false) - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - // filter the output via condition - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |$eval - |${ev.code} - |$existsVar = !${ev.isNull} && ${ev.value}; - """.stripMargin - } else { - s"$existsVar = true;" - } - - val resultVar = input ++ Seq(ExprCode.forNonNullValue( - JavaCode.variable(existsVar, BooleanType))) - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |boolean $existsVar = false; - |if ($matched != null) { - | $checkCondition - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $existsVar = false; - |if ($matches != null) { - | while (!$existsVar && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition - | } - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } - } - - protected def prepareRelation(ctx: CodegenContext): HashedRelationInfo -} - -object HashJoin extends CastSupport with SQLConfHelper { - - private def canRewriteAsLongType(keys: Seq[Expression]): Boolean = { - // TODO: support BooleanType, DateType and TimestampType - keys.forall(_.dataType.isInstanceOf[IntegralType]) && - keys.map(_.dataType.defaultSize).sum <= 8 - } - - /** - * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long. - * - * If not, returns the original expressions. - */ - def rewriteKeyExpr(keys: Seq[Expression]): Seq[Expression] = { - assert(keys.nonEmpty) - if (!canRewriteAsLongType(keys)) { - return keys - } - - var keyExpr: Expression = if (keys.head.dataType != LongType) { - cast(keys.head, LongType) - } else { - keys.head - } - keys.tail.foreach { e => - val bits = e.dataType.defaultSize * 8 - keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)), - BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1))) - } - keyExpr :: Nil - } - - /** - * Extract a given key which was previously packed in a long value using its index to - * determine the number of bits to shift - */ - def extractKeyExprAt(keys: Seq[Expression], index: Int): Expression = { - assert(canRewriteAsLongType(keys)) - // jump over keys that have a higher index value than the required key - if (keys.size == 1) { - assert(index == 0) - Cast( - child = BoundReference(0, LongType, nullable = false), - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } else { - val shiftedBits = - keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum - val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1 - // build the schema for unpacking the required key - val castChild = BitwiseAnd( - ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)), - Literal(mask)) - Cast( - child = castChild, - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } - } -} diff --git a/shims/spark34/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated b/shims/spark34/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated deleted file mode 100644 index 3fc823730614..000000000000 --- a/shims/spark34/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated +++ /dev/null @@ -1,774 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} -import org.apache.spark.sql.catalyst.analysis.CastSupport -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences -import org.apache.spark.sql.catalyst.expressions.codegen._ -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.physical.Partitioning -import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, RowIterator} -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.types.{BooleanType, IntegralType, LongType} - -/** - * @param relationTerm variable name for HashedRelation - * @param keyIsUnique indicate whether keys of HashedRelation known to be unique in code-gen time - * @param isEmpty indicate whether it known to be EmptyHashedRelation in code-gen time - */ -private[joins] case class HashedRelationInfo( - relationTerm: String, - keyIsUnique: Boolean, - isEmpty: Boolean) - -trait HashJoin extends JoinCodegenSupport { - def buildSide: BuildSide - - override def simpleStringWithNodeId(): String = { - val opId = ExplainUtils.getOpId(this) - s"$nodeName $joinType ${buildSide} ($opId)".trim - } - - override def output: Seq[Attribute] = { - joinType match { - case _: InnerLike => - left.output ++ right.output - case LeftOuter => - left.output ++ right.output.map(_.withNullability(true)) - case RightOuter => - left.output.map(_.withNullability(true)) ++ right.output - case j: ExistenceJoin => - left.output :+ j.exists - case LeftExistence(_) => - left.output - case x => - throw new IllegalArgumentException(s"HashJoin should not take $x as the JoinType") - } - } - - override def outputPartitioning: Partitioning = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - /** - * Handle the special cases for LeftOuter/LeftSemi with BuildLeft and RightOuter with BuildRight. - */ - override def outputOrdering: Seq[SortOrder] = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputOrdering - case LeftOuter => left.outputOrdering - case LeftSemi => left.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputOrdering - case RightOuter => right.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - protected lazy val (buildPlan, streamedPlan) = buildSide match { - case BuildLeft => (left, right) - case BuildRight => (right, left) - } - - protected lazy val (buildKeys, streamedKeys) = { - require(leftKeys.length == rightKeys.length && - leftKeys.map(_.dataType) - .zip(rightKeys.map(_.dataType)) - .forall(types => types._1.sameType(types._2)), - "Join keys from two sides should have same length and types") - buildSide match { - case BuildLeft => (leftKeys, rightKeys) - case BuildRight => (rightKeys, leftKeys) - } - } - - @transient protected lazy val (buildOutput, streamedOutput) = { - buildSide match { - case BuildLeft => (left.output, right.output) - case BuildRight => (right.output, left.output) - } - } - - @transient protected lazy val buildBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(buildKeys), buildOutput) - - @transient protected lazy val streamedBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(streamedKeys), streamedOutput) - - protected def buildSideKeyGenerator(): Projection = - UnsafeProjection.create(buildBoundKeys) - - protected def streamSideKeyGenerator(): UnsafeProjection = - UnsafeProjection.create(streamedBoundKeys) - - @transient protected[this] lazy val boundCondition = if (condition.isDefined) { - if (joinType == FullOuter && buildSide == BuildLeft) { - // Put join left side before right side. This is to be consistent with - // `ShuffledHashJoinExec.fullOuterJoin`. - Predicate.create(condition.get, buildPlan.output ++ streamedPlan.output).eval _ - } else { - Predicate.create(condition.get, streamedPlan.output ++ buildPlan.output).eval _ - } - } else { - (r: InternalRow) => true - } - - protected def createResultProjection(): (InternalRow) => InternalRow = joinType match { - case LeftExistence(_) => - UnsafeProjection.create(output, output) - case _ => - // Always put the stream side on left to simplify implementation - // both of left and right side could be null - UnsafeProjection.create( - output, (streamedPlan.output ++ buildPlan.output).map(_.withNullability(true))) - } - - private def innerJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinRow = new JoinedRow - val joinKeys = streamSideKeyGenerator() - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matched = hashedRelation.getValue(joinKeys(srow)) - if (matched != null) { - Some(joinRow.withRight(matched)).filter(boundCondition) - } else { - None - } - } - } else { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matches = hashedRelation.get(joinKeys(srow)) - if (matches != null) { - matches.map(joinRow.withRight).filter(boundCondition) - } else { - Seq.empty - } - } - } - } - - private def outerJoin( - streamedIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinedRow = new JoinedRow() - val keyGenerator = streamSideKeyGenerator() - val nullRow = new GenericInternalRow(buildPlan.output.length) - - if (hashedRelation.keyIsUnique) { - streamedIter.map { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val matched = hashedRelation.getValue(rowKey) - if (matched != null && boundCondition(joinedRow.withRight(matched))) { - joinedRow - } else { - joinedRow.withRight(nullRow) - } - } - } else { - streamedIter.flatMap { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val buildIter = hashedRelation.get(rowKey) - new RowIterator { - private var found = false - override def advanceNext(): Boolean = { - while (buildIter != null && buildIter.hasNext) { - val nextBuildRow = buildIter.next() - if (boundCondition(joinedRow.withRight(nextBuildRow))) { - found = true - return true - } - } - if (!found) { - joinedRow.withRight(nullRow) - found = true - return true - } - false - } - override def getRow: InternalRow = joinedRow - }.toScala - } - } - } - - private def semiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - } - } - } - - private def existenceJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val result = new GenericInternalRow(Array[Any](null)) - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.map { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - val exists = !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } else { - streamIter.map { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - val exists = !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } - } - - private def antiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - // If the right side is empty, AntiJoin simply returns the left side. - if (hashedRelation == EmptyHashedRelation) { - return streamIter - } - - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - key.anyNull || matched == null || - (condition.isDefined && !boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - key.anyNull || buildIter == null || (condition.isDefined && !buildIter.exists { - row => boundCondition(joinedRow(current, row)) - }) - } - } - } - - protected def join( - streamedIter: Iterator[InternalRow], - hashed: HashedRelation, - numOutputRows: SQLMetric): Iterator[InternalRow] = { - - val joinedIter = joinType match { - case _: InnerLike => - innerJoin(streamedIter, hashed) - case LeftOuter | RightOuter => - outerJoin(streamedIter, hashed) - case LeftSemi => - semiJoin(streamedIter, hashed) - case LeftAnti => - antiJoin(streamedIter, hashed) - case _: ExistenceJoin => - existenceJoin(streamedIter, hashed) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - - val resultProj = createResultProjection - joinedIter.map { r => - numOutputRows += 1 - resultProj(r) - } - } - - override def doProduce(ctx: CodegenContext): String = { - streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this) - } - - override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - joinType match { - case _: InnerLike => codegenInner(ctx, input) - case LeftOuter | RightOuter => codegenOuter(ctx, input) - case LeftSemi => codegenSemi(ctx, input) - case LeftAnti => codegenAnti(ctx, input) - case _: ExistenceJoin => codegenExistence(ctx, input) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - } - - /** - * Returns the code for generating join key for stream side, and expression of whether the key - * has any null in it or not. - */ - protected def genStreamSideJoinKey( - ctx: CodegenContext, - input: Seq[ExprCode]): (ExprCode, String) = { - ctx.currentVars = input - if (streamedBoundKeys.length == 1 && streamedBoundKeys.head.dataType == LongType) { - // generate the join key as Long - val ev = streamedBoundKeys.head.genCode(ctx) - (ev, ev.isNull) - } else { - // generate the join key as UnsafeRow - val ev = GenerateUnsafeProjection.createCode(ctx, streamedBoundKeys) - (ev, s"${ev.value}.anyNull()") - } - } - - /** - * Generates the code for Inner join. - */ - protected def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, buildVars) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash inner join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? - | null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | while ($matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left or right outer join. - */ - protected def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val matched = ctx.freshName("matched") - val buildVars = genOneSideJoinVars(ctx, matched, buildPlan, setDefaultValue = true) - val numOutput = metricTerm(ctx, "numOutputRows") - - // filter the output via condition - val conditionPassed = ctx.freshName("conditionPassed") - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |boolean $conditionPassed = true; - |${eval.trim} - |if ($matched != null) { - | ${ev.code} - | $conditionPassed = !${ev.isNull} && ${ev.value}; - |} - """.stripMargin - } else { - s"final boolean $conditionPassed = true;" - } - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |${checkCondition.trim} - |if (!$conditionPassed) { - | $matched = null; - | // reset the variables those are already evaluated. - | ${buildVars.filter(_.code.isEmpty).map(v => s"${v.isNull} = true;").mkString("\n")} - |} - |$numOutput.add(1); - |${consume(ctx, resultVars)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $found = false; - |// the last iteration of this loop is to emit an empty row if there is no matched rows. - |while ($matches != null && $matches.hasNext() || !$found) { - | UnsafeRow $matched = $matches != null && $matches.hasNext() ? - | (UnsafeRow) $matches.next() : null; - | ${checkCondition.trim} - | if ($conditionPassed) { - | $found = true; - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left semi join. - */ - protected def codegenSemi(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash semi join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | boolean $found = false; - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | if ($found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for anti join. - */ - protected def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val numOutput = metricTerm(ctx, "numOutputRows") - if (isEmptyHashedRelation) { - return s""" - |// If HashedRelation is empty, hash anti join simply returns the stream side. - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } - - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - - if (keyIsUnique) { - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); - | if ($matches != null) { - | // Evaluate the condition. - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } - } - - /** - * Generates the code for existence join. - */ - protected def codegenExistence(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val numOutput = metricTerm(ctx, "numOutputRows") - val existsVar = ctx.freshName("exists") - - val matched = ctx.freshName("matched") - val buildVars = genOneSideJoinVars(ctx, matched, buildPlan, setDefaultValue = false) - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - // filter the output via condition - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |$eval - |${ev.code} - |$existsVar = !${ev.isNull} && ${ev.value}; - """.stripMargin - } else { - s"$existsVar = true;" - } - - val resultVar = input ++ Seq(ExprCode.forNonNullValue( - JavaCode.variable(existsVar, BooleanType))) - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |boolean $existsVar = false; - |if ($matched != null) { - | $checkCondition - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $existsVar = false; - |if ($matches != null) { - | while (!$existsVar && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition - | } - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } - } - - protected def prepareRelation(ctx: CodegenContext): HashedRelationInfo -} - -object HashJoin extends CastSupport with SQLConfHelper { - - private def canRewriteAsLongType(keys: Seq[Expression]): Boolean = { - // TODO: support BooleanType, DateType and TimestampType - keys.forall(_.dataType.isInstanceOf[IntegralType]) && - keys.map(_.dataType.defaultSize).sum <= 8 - } - - /** - * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long. - * - * If not, returns the original expressions. - */ - def rewriteKeyExpr(keys: Seq[Expression]): Seq[Expression] = { - assert(keys.nonEmpty) - if (!canRewriteAsLongType(keys)) { - return keys - } - - var keyExpr: Expression = if (keys.head.dataType != LongType) { - cast(keys.head, LongType) - } else { - keys.head - } - keys.tail.foreach { e => - val bits = e.dataType.defaultSize * 8 - keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)), - BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1))) - } - keyExpr :: Nil - } - - /** - * Extract a given key which was previously packed in a long value using its index to - * determine the number of bits to shift - */ - def extractKeyExprAt(keys: Seq[Expression], index: Int): Expression = { - assert(canRewriteAsLongType(keys)) - // jump over keys that have a higher index value than the required key - if (keys.size == 1) { - assert(index == 0) - Cast( - child = BoundReference(0, LongType, nullable = false), - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } else { - val shiftedBits = - keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum - val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1 - // build the schema for unpacking the required key - val castChild = BitwiseAnd( - ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)), - Literal(mask)) - Cast( - child = castChild, - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } - } -} diff --git a/shims/spark35/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated b/shims/spark35/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated deleted file mode 100644 index 3fc823730614..000000000000 --- a/shims/spark35/src/main/java/org/apache/spark/sql/execution/joins/HashJoin.scala.deprecated +++ /dev/null @@ -1,774 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.joins - -import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} -import org.apache.spark.sql.catalyst.analysis.CastSupport -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences -import org.apache.spark.sql.catalyst.expressions.codegen._ -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.physical.Partitioning -import org.apache.spark.sql.execution.{CodegenSupport, ExplainUtils, RowIterator} -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.types.{BooleanType, IntegralType, LongType} - -/** - * @param relationTerm variable name for HashedRelation - * @param keyIsUnique indicate whether keys of HashedRelation known to be unique in code-gen time - * @param isEmpty indicate whether it known to be EmptyHashedRelation in code-gen time - */ -private[joins] case class HashedRelationInfo( - relationTerm: String, - keyIsUnique: Boolean, - isEmpty: Boolean) - -trait HashJoin extends JoinCodegenSupport { - def buildSide: BuildSide - - override def simpleStringWithNodeId(): String = { - val opId = ExplainUtils.getOpId(this) - s"$nodeName $joinType ${buildSide} ($opId)".trim - } - - override def output: Seq[Attribute] = { - joinType match { - case _: InnerLike => - left.output ++ right.output - case LeftOuter => - left.output ++ right.output.map(_.withNullability(true)) - case RightOuter => - left.output.map(_.withNullability(true)) ++ right.output - case j: ExistenceJoin => - left.output :+ j.exists - case LeftExistence(_) => - left.output - case x => - throw new IllegalArgumentException(s"HashJoin should not take $x as the JoinType") - } - } - - override def outputPartitioning: Partitioning = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputPartitioning - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - /** - * Handle the special cases for LeftOuter/LeftSemi with BuildLeft and RightOuter with BuildRight. - */ - override def outputOrdering: Seq[SortOrder] = buildSide match { - case BuildLeft => - joinType match { - case _: InnerLike | RightOuter => right.outputOrdering - case LeftOuter => left.outputOrdering - case LeftSemi => left.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building left side") - } - case BuildRight => - joinType match { - case _: InnerLike | LeftOuter | LeftSemi | LeftAnti | _: ExistenceJoin => - left.outputOrdering - case RightOuter => right.outputOrdering - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType with building right side") - } - } - - protected lazy val (buildPlan, streamedPlan) = buildSide match { - case BuildLeft => (left, right) - case BuildRight => (right, left) - } - - protected lazy val (buildKeys, streamedKeys) = { - require(leftKeys.length == rightKeys.length && - leftKeys.map(_.dataType) - .zip(rightKeys.map(_.dataType)) - .forall(types => types._1.sameType(types._2)), - "Join keys from two sides should have same length and types") - buildSide match { - case BuildLeft => (leftKeys, rightKeys) - case BuildRight => (rightKeys, leftKeys) - } - } - - @transient protected lazy val (buildOutput, streamedOutput) = { - buildSide match { - case BuildLeft => (left.output, right.output) - case BuildRight => (right.output, left.output) - } - } - - @transient protected lazy val buildBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(buildKeys), buildOutput) - - @transient protected lazy val streamedBoundKeys = - bindReferences(HashJoin.rewriteKeyExpr(streamedKeys), streamedOutput) - - protected def buildSideKeyGenerator(): Projection = - UnsafeProjection.create(buildBoundKeys) - - protected def streamSideKeyGenerator(): UnsafeProjection = - UnsafeProjection.create(streamedBoundKeys) - - @transient protected[this] lazy val boundCondition = if (condition.isDefined) { - if (joinType == FullOuter && buildSide == BuildLeft) { - // Put join left side before right side. This is to be consistent with - // `ShuffledHashJoinExec.fullOuterJoin`. - Predicate.create(condition.get, buildPlan.output ++ streamedPlan.output).eval _ - } else { - Predicate.create(condition.get, streamedPlan.output ++ buildPlan.output).eval _ - } - } else { - (r: InternalRow) => true - } - - protected def createResultProjection(): (InternalRow) => InternalRow = joinType match { - case LeftExistence(_) => - UnsafeProjection.create(output, output) - case _ => - // Always put the stream side on left to simplify implementation - // both of left and right side could be null - UnsafeProjection.create( - output, (streamedPlan.output ++ buildPlan.output).map(_.withNullability(true))) - } - - private def innerJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinRow = new JoinedRow - val joinKeys = streamSideKeyGenerator() - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matched = hashedRelation.getValue(joinKeys(srow)) - if (matched != null) { - Some(joinRow.withRight(matched)).filter(boundCondition) - } else { - None - } - } - } else { - streamIter.flatMap { srow => - joinRow.withLeft(srow) - val matches = hashedRelation.get(joinKeys(srow)) - if (matches != null) { - matches.map(joinRow.withRight).filter(boundCondition) - } else { - Seq.empty - } - } - } - } - - private def outerJoin( - streamedIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinedRow = new JoinedRow() - val keyGenerator = streamSideKeyGenerator() - val nullRow = new GenericInternalRow(buildPlan.output.length) - - if (hashedRelation.keyIsUnique) { - streamedIter.map { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val matched = hashedRelation.getValue(rowKey) - if (matched != null && boundCondition(joinedRow.withRight(matched))) { - joinedRow - } else { - joinedRow.withRight(nullRow) - } - } - } else { - streamedIter.flatMap { currentRow => - val rowKey = keyGenerator(currentRow) - joinedRow.withLeft(currentRow) - val buildIter = hashedRelation.get(rowKey) - new RowIterator { - private var found = false - override def advanceNext(): Boolean = { - while (buildIter != null && buildIter.hasNext) { - val nextBuildRow = buildIter.next() - if (boundCondition(joinedRow.withRight(nextBuildRow))) { - found = true - return true - } - } - if (!found) { - joinedRow.withRight(nullRow) - found = true - return true - } - false - } - override def getRow: InternalRow = joinedRow - }.toScala - } - } - } - - private def semiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation == EmptyHashedRelation) { - Iterator.empty - } else if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - } - } - } - - private def existenceJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - val joinKeys = streamSideKeyGenerator() - val result = new GenericInternalRow(Array[Any](null)) - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.map { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - val exists = !key.anyNull && matched != null && - (condition.isEmpty || boundCondition(joinedRow(current, matched))) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } else { - streamIter.map { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - val exists = !key.anyNull && buildIter != null && (condition.isEmpty || buildIter.exists { - (row: InternalRow) => boundCondition(joinedRow(current, row)) - }) - result.setBoolean(0, exists) - joinedRow(current, result) - } - } - } - - private def antiJoin( - streamIter: Iterator[InternalRow], - hashedRelation: HashedRelation): Iterator[InternalRow] = { - // If the right side is empty, AntiJoin simply returns the left side. - if (hashedRelation == EmptyHashedRelation) { - return streamIter - } - - val joinKeys = streamSideKeyGenerator() - val joinedRow = new JoinedRow - - if (hashedRelation.keyIsUnique) { - streamIter.filter { current => - val key = joinKeys(current) - lazy val matched = hashedRelation.getValue(key) - key.anyNull || matched == null || - (condition.isDefined && !boundCondition(joinedRow(current, matched))) - } - } else { - streamIter.filter { current => - val key = joinKeys(current) - lazy val buildIter = hashedRelation.get(key) - key.anyNull || buildIter == null || (condition.isDefined && !buildIter.exists { - row => boundCondition(joinedRow(current, row)) - }) - } - } - } - - protected def join( - streamedIter: Iterator[InternalRow], - hashed: HashedRelation, - numOutputRows: SQLMetric): Iterator[InternalRow] = { - - val joinedIter = joinType match { - case _: InnerLike => - innerJoin(streamedIter, hashed) - case LeftOuter | RightOuter => - outerJoin(streamedIter, hashed) - case LeftSemi => - semiJoin(streamedIter, hashed) - case LeftAnti => - antiJoin(streamedIter, hashed) - case _: ExistenceJoin => - existenceJoin(streamedIter, hashed) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - - val resultProj = createResultProjection - joinedIter.map { r => - numOutputRows += 1 - resultProj(r) - } - } - - override def doProduce(ctx: CodegenContext): String = { - streamedPlan.asInstanceOf[CodegenSupport].produce(ctx, this) - } - - override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = { - joinType match { - case _: InnerLike => codegenInner(ctx, input) - case LeftOuter | RightOuter => codegenOuter(ctx, input) - case LeftSemi => codegenSemi(ctx, input) - case LeftAnti => codegenAnti(ctx, input) - case _: ExistenceJoin => codegenExistence(ctx, input) - case x => - throw new IllegalArgumentException( - s"HashJoin should not take $x as the JoinType") - } - } - - /** - * Returns the code for generating join key for stream side, and expression of whether the key - * has any null in it or not. - */ - protected def genStreamSideJoinKey( - ctx: CodegenContext, - input: Seq[ExprCode]): (ExprCode, String) = { - ctx.currentVars = input - if (streamedBoundKeys.length == 1 && streamedBoundKeys.head.dataType == LongType) { - // generate the join key as Long - val ev = streamedBoundKeys.head.genCode(ctx) - (ev, ev.isNull) - } else { - // generate the join key as UnsafeRow - val ev = GenerateUnsafeProjection.createCode(ctx, streamedBoundKeys) - (ev, s"${ev.value}.anyNull()") - } - } - - /** - * Generates the code for Inner join. - */ - protected def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, buildVars) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash inner join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? - | null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | while ($matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left or right outer join. - */ - protected def codegenOuter(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val matched = ctx.freshName("matched") - val buildVars = genOneSideJoinVars(ctx, matched, buildPlan, setDefaultValue = true) - val numOutput = metricTerm(ctx, "numOutputRows") - - // filter the output via condition - val conditionPassed = ctx.freshName("conditionPassed") - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |boolean $conditionPassed = true; - |${eval.trim} - |if ($matched != null) { - | ${ev.code} - | $conditionPassed = !${ev.isNull} && ${ev.value}; - |} - """.stripMargin - } else { - s"final boolean $conditionPassed = true;" - } - - val resultVars = buildSide match { - case BuildLeft => buildVars ++ input - case BuildRight => input ++ buildVars - } - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |${checkCondition.trim} - |if (!$conditionPassed) { - | $matched = null; - | // reset the variables those are already evaluated. - | ${buildVars.filter(_.code.isEmpty).map(v => s"${v.isNull} = true;").mkString("\n")} - |} - |$numOutput.add(1); - |${consume(ctx, resultVars)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $found = false; - |// the last iteration of this loop is to emit an empty row if there is no matched rows. - |while ($matches != null && $matches.hasNext() || !$found) { - | UnsafeRow $matched = $matches != null && $matches.hasNext() ? - | (UnsafeRow) $matches.next() : null; - | ${checkCondition.trim} - | if ($conditionPassed) { - | $found = true; - | $numOutput.add(1); - | ${consume(ctx, resultVars)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for left semi join. - */ - protected def codegenSemi(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - val numOutput = metricTerm(ctx, "numOutputRows") - - if (isEmptyHashedRelation) { - """ - |// If HashedRelation is empty, hash semi join simply returns nothing. - """.stripMargin - } else if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |if ($matched != null) { - | $checkCondition { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |if ($matches != null) { - | boolean $found = false; - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | if ($found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - | } - |} - """.stripMargin - } - } - - /** - * Generates the code for anti join. - */ - protected def codegenAnti(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, isEmptyHashedRelation) = prepareRelation(ctx) - val numOutput = metricTerm(ctx, "numOutputRows") - if (isEmptyHashedRelation) { - return s""" - |// If HashedRelation is empty, hash anti join simply returns the stream side. - |$numOutput.add(1); - |${consume(ctx, input)} - """.stripMargin - } - - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val (matched, checkCondition, _) = getJoinCondition(ctx, input, streamedPlan, buildPlan) - - if (keyIsUnique) { - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | UnsafeRow $matched = (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - | if ($matched != null) { - | // Evaluate the condition. - | $checkCondition { - | $found = true; - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - val found = ctx.freshName("found") - s""" - |boolean $found = false; - |// generate join key for stream side - |${keyEv.code} - |// Check if the key has nulls. - |if (!($anyNull)) { - | // Check if the HashedRelation exists. - | $iteratorCls $matches = ($iteratorCls)$relationTerm.get(${keyEv.value}); - | if ($matches != null) { - | // Evaluate the condition. - | while (!$found && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition { - | $found = true; - | } - | } - | } - |} - |if (!$found) { - | $numOutput.add(1); - | ${consume(ctx, input)} - |} - """.stripMargin - } - } - - /** - * Generates the code for existence join. - */ - protected def codegenExistence(ctx: CodegenContext, input: Seq[ExprCode]): String = { - val HashedRelationInfo(relationTerm, keyIsUnique, _) = prepareRelation(ctx) - val (keyEv, anyNull) = genStreamSideJoinKey(ctx, input) - val numOutput = metricTerm(ctx, "numOutputRows") - val existsVar = ctx.freshName("exists") - - val matched = ctx.freshName("matched") - val buildVars = genOneSideJoinVars(ctx, matched, buildPlan, setDefaultValue = false) - val checkCondition = if (condition.isDefined) { - val expr = condition.get - // evaluate the variables from build side that used by condition - val eval = evaluateRequiredVariables(buildPlan.output, buildVars, expr.references) - // filter the output via condition - ctx.currentVars = input ++ buildVars - val ev = - BindReferences.bindReference(expr, streamedPlan.output ++ buildPlan.output).genCode(ctx) - s""" - |$eval - |${ev.code} - |$existsVar = !${ev.isNull} && ${ev.value}; - """.stripMargin - } else { - s"$existsVar = true;" - } - - val resultVar = input ++ Seq(ExprCode.forNonNullValue( - JavaCode.variable(existsVar, BooleanType))) - - if (keyIsUnique) { - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashedRelation - |UnsafeRow $matched = $anyNull ? null: (UnsafeRow)$relationTerm.getValue(${keyEv.value}); - |boolean $existsVar = false; - |if ($matched != null) { - | $checkCondition - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } else { - val matches = ctx.freshName("matches") - val iteratorCls = classOf[Iterator[UnsafeRow]].getName - s""" - |// generate join key for stream side - |${keyEv.code} - |// find matches from HashRelation - |$iteratorCls $matches = $anyNull ? null : ($iteratorCls)$relationTerm.get(${keyEv.value}); - |boolean $existsVar = false; - |if ($matches != null) { - | while (!$existsVar && $matches.hasNext()) { - | UnsafeRow $matched = (UnsafeRow) $matches.next(); - | $checkCondition - | } - |} - |$numOutput.add(1); - |${consume(ctx, resultVar)} - """.stripMargin - } - } - - protected def prepareRelation(ctx: CodegenContext): HashedRelationInfo -} - -object HashJoin extends CastSupport with SQLConfHelper { - - private def canRewriteAsLongType(keys: Seq[Expression]): Boolean = { - // TODO: support BooleanType, DateType and TimestampType - keys.forall(_.dataType.isInstanceOf[IntegralType]) && - keys.map(_.dataType.defaultSize).sum <= 8 - } - - /** - * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long. - * - * If not, returns the original expressions. - */ - def rewriteKeyExpr(keys: Seq[Expression]): Seq[Expression] = { - assert(keys.nonEmpty) - if (!canRewriteAsLongType(keys)) { - return keys - } - - var keyExpr: Expression = if (keys.head.dataType != LongType) { - cast(keys.head, LongType) - } else { - keys.head - } - keys.tail.foreach { e => - val bits = e.dataType.defaultSize * 8 - keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)), - BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1))) - } - keyExpr :: Nil - } - - /** - * Extract a given key which was previously packed in a long value using its index to - * determine the number of bits to shift - */ - def extractKeyExprAt(keys: Seq[Expression], index: Int): Expression = { - assert(canRewriteAsLongType(keys)) - // jump over keys that have a higher index value than the required key - if (keys.size == 1) { - assert(index == 0) - Cast( - child = BoundReference(0, LongType, nullable = false), - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } else { - val shiftedBits = - keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum - val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1 - // build the schema for unpacking the required key - val castChild = BitwiseAnd( - ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)), - Literal(mask)) - Cast( - child = castChild, - dataType = keys(index).dataType, - timeZoneId = Option(conf.sessionLocalTimeZone), - ansiEnabled = false) - } - } -} From e870de8cea51d138d49e675e1031f090bfc5bf19 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Fri, 31 May 2024 16:55:53 +0800 Subject: [PATCH 187/402] [VL] Fall back collect_set, min and max when input is complex type (#5934) [VL] Fall back collect_set, min and max when input is complex type. --- .../VeloxAggregateFunctionsSuite.scala | 23 +++++++++++++++++++ .../SubstraitToVeloxPlanValidator.cc | 10 ++++++++ 2 files changed, 33 insertions(+) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index 4f6f4eb224d0..ae6306cc0d4a 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -22,7 +22,9 @@ import org.apache.gluten.extension.columnar.validator.FallbackInjects import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial} import org.apache.spark.sql.execution.aggregate.BaseAggregateExec +import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types._ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSuite { @@ -1112,6 +1114,27 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu } } } + + test("complex type with null") { + val jsonStr = """{"txn":{"appId":"txnId","version":0,"lastUpdated":null}}""" + val jsonSchema = StructType( + Seq( + StructField( + "txn", + StructType( + Seq( + StructField("appId", StringType, true), + StructField("lastUpdated", LongType, true), + StructField("version", LongType, true))), + true))) + val df = spark.read.schema(jsonSchema).json(Seq(jsonStr).toDS) + df.select(collect_set(col("txn"))).collect + + df.select(min(col("txn"))).collect + + df.select(max(col("txn"))).collect + + } } class VeloxAggregateFunctionsDefaultSuite extends VeloxAggregateFunctionsSuite { diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index abb2bbc560f4..a3b46d7d08e1 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -1045,6 +1045,16 @@ bool SubstraitToVeloxPlanValidator::validateAggRelFunctionType(const ::substrait LOG_VALIDATION_MSG("Validation failed for function " + funcName + " resolve type in AggregateRel."); return false; } + static const std::unordered_set notSupportComplexTypeAggFuncs = {"set_agg", "min", "max"}; + if (notSupportComplexTypeAggFuncs.find(baseFuncName) != notSupportComplexTypeAggFuncs.end() && + exec::isRawInput(funcStep)) { + auto type = binder.tryResolveType(signature->argumentTypes()[0]); + if (type->isArray() || type->isMap() || type->isRow()) { + LOG_VALIDATION_MSG("Validation failed for function " + baseFuncName + " complex type is not supported."); + return false; + } + } + resolved = true; break; } From c94cde4ffd479de8a1b09cbebaa46f9344cdb2e6 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Fri, 31 May 2024 17:41:38 +0800 Subject: [PATCH 188/402] [GLUTEN-5896][CH]Fix greatest diff #5920 What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #5896) How was this patch tested? TEST BY UT --- ...enClickHouseTPCHSaltNullParquetSuite.scala | 12 +++ .../Functions/SparkFunctionGreatest.cpp | 75 +++++++++++++++++++ .../Parser/SerializedPlanParser.h | 2 +- 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index eec0ad874c5d..748bd5a7f7f6 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2551,5 +2551,17 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) spark.sql("drop table test_tbl_5096") } + + test("GLUTEN-5896: Bug fix greatest diff") { + val tbl_create_sql = + "create table test_tbl_5896(id bigint, x1 int, x2 int, x3 int) using parquet" + val tbl_insert_sql = + "insert into test_tbl_5896 values(1, 12, NULL, 13), (2, NULL, NULL, NULL), (3, 11, NULL, NULL), (4, 10, 9, 8)" + val select_sql = "select id, greatest(x1, x2, x3) from test_tbl_5896" + spark.sql(tbl_create_sql) + spark.sql(tbl_insert_sql) + compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) + spark.sql("drop table test_tbl_5896") + } } // scalastyle:on line.size.limit diff --git a/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp b/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp new file mode 100644 index 000000000000..9577d65ec5f7 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class SparkFunctionGreatest : public DB::FunctionLeastGreatestGeneric +{ +public: + static constexpr auto name = "sparkGreatest"; + static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared(); } + SparkFunctionGreatest() = default; + ~SparkFunctionGreatest() override = default; + bool useDefaultImplementationForNulls() const override { return false; } + +private: + DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes & types) const override + { + if (types.empty()) + throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} cannot be called without arguments", name); + return makeNullable(getLeastSupertype(types)); + } + + DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows_count) const override + { + size_t num_arguments = arguments.size(); + DB::Columns converted_columns(num_arguments); + for (size_t arg = 0; arg < num_arguments; ++arg) + converted_columns[arg] = castColumn(arguments[arg], result_type)->convertToFullColumnIfConst(); + auto result_column = result_type->createColumn(); + result_column->reserve(input_rows_count); + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + size_t best_arg = 0; + for (size_t arg = 1; arg < num_arguments; ++arg) + { + auto cmp_result = converted_columns[arg]->compareAt(row_num, row_num, *converted_columns[best_arg], -1); + if (cmp_result > 0) + best_arg = arg; + } + result_column->insertFrom(*converted_columns[best_arg], row_num); + } + return result_column; + } +}; + +REGISTER_FUNCTION(SparkGreatest) +{ + factory.registerFunction(); +} +} diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index a636ebb9352f..73448b0690c2 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -104,7 +104,7 @@ static const std::map SCALAR_FUNCTIONS {"hypot", "hypot"}, {"sign", "sign"}, {"radians", "radians"}, - {"greatest", "greatest"}, + {"greatest", "sparkGreatest"}, {"least", "least"}, {"shiftleft", "bitShiftLeft"}, {"shiftright", "bitShiftRight"}, From 2fc808d273712248a142f4448bc7e79285725014 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Fri, 31 May 2024 18:06:13 +0800 Subject: [PATCH 189/402] [CORE] Remove IteratorApi.genNativeFileScanRDD, both velox and clickchouse backend needn't it. (#5937) [CORE] Remove IteratorApi.genNativeFileScanRDD, both velox and clickchouse backend needn't it. --- .../clickhouse/CHIteratorApi.scala | 43 +-- .../benchmarks/CHParquetReadBenchmark.scala | 255 ------------------ .../backendsapi/velox/VeloxIteratorApi.scala | 15 +- .../gluten/backendsapi/IteratorApi.scala | 11 - .../execution/BasicScanExecTransformer.scala | 32 +-- .../execution/BatchScanExecTransformer.scala | 6 - .../FileSourceScanExecTransformer.scala | 6 - .../hive/HiveTableScanExecTransformer.scala | 6 - 8 files changed, 6 insertions(+), 368 deletions(-) delete mode 100644 backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 4926a97eb824..bc16c2d77fe1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -20,17 +20,16 @@ import org.apache.gluten.{GlutenConfig, GlutenNumaBindingInfo} import org.apache.gluten.backendsapi.IteratorApi import org.apache.gluten.execution._ import org.apache.gluten.expression.ConverterUtils -import org.apache.gluten.metrics.{GlutenTimeMetric, IMetrics, NativeMetrics} +import org.apache.gluten.metrics.{IMetrics, NativeMetrics} import org.apache.gluten.substrait.plan.PlanNode import org.apache.gluten.substrait.rel._ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.utils.LogLevelUtil import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator, GeneralOutIterator} -import org.apache.spark.{InterruptibleIterator, SparkConf, SparkContext, TaskContext} +import org.apache.spark.{InterruptibleIterator, SparkConf, TaskContext} import org.apache.spark.affinity.CHAffinity import org.apache.spark.internal.Logging -import org.apache.spark.rdd.RDD import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.datasources.FilePartition import org.apache.spark.sql.execution.metric.SQLMetric @@ -315,44 +314,6 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { context.addTaskCompletionListener[Unit](_ => close()) new CloseableCHColumnBatchIterator(resIter, Some(pipelineTime)) } - - /** Generate Native FileScanRDD, currently only for ClickHouse Backend. */ - override def genNativeFileScanRDD( - sparkContext: SparkContext, - wsCtx: WholeStageTransformContext, - splitInfos: Seq[SplitInfo], - scan: BasicScanExecTransformer, - numOutputRows: SQLMetric, - numOutputBatches: SQLMetric, - scanTime: SQLMetric): RDD[ColumnarBatch] = { - val substraitPlanPartition = GlutenTimeMetric.withMillisTime { - val planByteArray = wsCtx.root.toProtobuf.toByteArray - splitInfos.zipWithIndex.map { - case (splitInfo, index) => - val splitInfoByteArray = splitInfo match { - case filesNode: LocalFilesNode => - setFileSchemaForLocalFiles(filesNode, scan) - filesNode.setFileReadProperties(mapAsJavaMap(scan.getProperties)) - filesNode.toProtobuf.toByteArray - case extensionTableNode: ExtensionTableNode => - extensionTableNode.toProtobuf.toByteArray - } - - GlutenPartition( - index, - planByteArray, - Array(splitInfoByteArray), - locations = splitInfo.preferredLocations().asScala.toArray) - } - }(t => logInfo(s"Generating the Substrait plan took: $t ms.")) - - new NativeFileScanColumnarRDD( - sparkContext, - substraitPlanPartition, - numOutputRows, - numOutputBatches, - scanTime) - } } object CHIteratorApi { diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala deleted file mode 100644 index dc1431fa64fa..000000000000 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.benchmarks - -import org.apache.gluten.GlutenConfig -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformContext} -import org.apache.gluten.expression.ConverterUtils -import org.apache.gluten.sql.shims.SparkShimLoader -import org.apache.gluten.substrait.SubstraitContext -import org.apache.gluten.substrait.plan.PlanBuilder -import org.apache.gluten.vectorized.{CHBlockConverterJniWrapper, CHNativeBlock} - -import org.apache.spark.benchmark.Benchmark -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{UnsafeProjection, UnsafeRow} -import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark -import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile} -import org.apache.spark.sql.vectorized.ColumnarBatch - -import com.google.common.collect.Lists - -import scala.collection.JavaConverters._ - -/** - * Benchmark to measure Clickhouse parquet read performance. To run this benchmark: - * {{{ - * 1. Run in IDEA: run this class directly; - * 2. Run without IDEA: bin/spark-submit --class - * --jars ,, - * --conf xxxx=xxx - * backends-clickhouse-XXX-tests.jar - * parameters - * - * Parameters: - * 1. parquet files dir; - * 2. the count of the parquet file to read; - * 3. the fields to read; - * 4. the execution count; - * 5. whether to run vanilla spark benchmarks; - * }}} - */ -object CHParquetReadBenchmark extends SqlBasedBenchmark with CHSqlBasedBenchmark { - - protected lazy val appName = "CHParquetReadBenchmark" - protected lazy val thrdNum = "1" - protected lazy val memorySize = "4G" - protected lazy val offheapSize = "4G" - - def beforeAll(): Unit = {} - - override def getSparkSession: SparkSession = { - beforeAll() - val conf = getSparkConf - .setIfMissing("spark.sql.columnVector.offheap.enabled", "true") - .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "true") - - SparkSession.builder.config(conf).getOrCreate() - } - - override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val (parquetDir, readFileCnt, scanSchema, executedCnt, executedVanilla) = - if (mainArgs.isEmpty) { - ("/data/tpch-data/parquet/lineitem", 3, "l_orderkey,l_receiptdate", 5, true) - } else { - (mainArgs(0), mainArgs(1).toInt, mainArgs(2), mainArgs(3).toInt, mainArgs(4).toBoolean) - } - - val chParquet = spark.sql(s""" - |select $scanSchema from parquet.`$parquetDir` - | - |""".stripMargin) - - // Get the `FileSourceScanExecTransformer` - val chScanPlan = chParquet.queryExecution.executedPlan.collect { - case scan: FileSourceScanExecTransformer => scan - } - - val chFileScan = chScanPlan.head - val outputAttrs = chFileScan.outputAttributes() - val filePartitions = chFileScan.getPartitions - .take(readFileCnt) - .map(_.asInstanceOf[FilePartition]) - - val numOutputRows = chFileScan.longMetric("numOutputRows") - val numOutputVectors = chFileScan.longMetric("outputVectors") - val scanTime = chFileScan.longMetric("scanTime") - // Generate Substrait plan - val substraitContext = new SubstraitContext - val transformContext = chFileScan.transform(substraitContext) - val outNames = new java.util.ArrayList[String]() - for (attr <- outputAttrs) { - outNames.add(ConverterUtils.genColumnNameWithExprId(attr)) - } - val planNode = - PlanBuilder.makePlan(substraitContext, Lists.newArrayList(transformContext.root), outNames) - - val nativeFileScanRDD = BackendsApiManager.getIteratorApiInstance.genNativeFileScanRDD( - spark.sparkContext, - WholeStageTransformContext(planNode, substraitContext), - chFileScan.getSplitInfos, - chFileScan, - numOutputRows, - numOutputVectors, - scanTime - ) - - // Get the total row count - val chRowCnt = nativeFileScanRDD - .mapPartitionsInternal(batches => batches.map(batch => batch.numRows().toLong)) - .collect() - .sum - - val parquetReadBenchmark = - new Benchmark( - s"Parquet Read $readFileCnt files, fields: $scanSchema, total $chRowCnt records", - chRowCnt, - output = output) - - parquetReadBenchmark.addCase(s"ClickHouse Parquet Read", executedCnt) { - _ => - val resultRDD: RDD[Long] = nativeFileScanRDD.mapPartitionsInternal { - batches => - batches.map { - batch => - val block = CHNativeBlock.fromColumnarBatch(batch) - block.totalBytes() - block.close() - batch.numRows().toLong - } - } - resultRDD.collect() - } - - parquetReadBenchmark.addCase(s"ClickHouse Parquet Read to Rows", executedCnt) { - _ => - val resultRDD: RDD[Long] = nativeFileScanRDD.mapPartitionsInternal { - batches => - batches.map { - batch => - val block = CHNativeBlock.fromColumnarBatch(batch) - val info = - CHBlockConverterJniWrapper.convertColumnarToRow(block.blockAddress(), null) - new Iterator[InternalRow] { - var rowId = 0 - val row = new UnsafeRow(batch.numCols()) - var closed = false - - override def hasNext: Boolean = { - val result = rowId < batch.numRows() - if (!result && !closed) { - CHBlockConverterJniWrapper.freeMemory(info.memoryAddress, info.totalSize) - closed = true - } - result - } - - override def next: UnsafeRow = { - if (rowId >= batch.numRows()) throw new NoSuchElementException - - val (offset, length) = (info.offsets(rowId), info.lengths(rowId)) - row.pointTo(null, info.memoryAddress + offset, length.toInt) - rowId += 1 - row - } - }.foreach(_.numFields) - block.close() - - batch.numRows().toLong - } - } - resultRDD.collect() - } - - if (executedVanilla) { - spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false") - - val vanillaParquet = spark.sql(s""" - |select $scanSchema from parquet.`$parquetDir` - | - |""".stripMargin) - - val vanillaScanPlan = vanillaParquet.queryExecution.executedPlan.collect { - case scan: FileSourceScanExec => scan - } - - val fileScan = vanillaScanPlan.head - val fileScanOutput = fileScan.output - val relation = fileScan.relation - val readFile: PartitionedFile => Iterator[InternalRow] = - relation.fileFormat.buildReaderWithPartitionValues( - sparkSession = relation.sparkSession, - dataSchema = relation.dataSchema, - partitionSchema = relation.partitionSchema, - requiredSchema = fileScan.requiredSchema, - filters = Seq.empty, - options = relation.options, - hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options) - ) - - val newFileScanRDD = - SparkShimLoader.getSparkShims - .generateFileScanRDD(spark, readFile, filePartitions, fileScan) - .asInstanceOf[RDD[ColumnarBatch]] - - val rowCnt = newFileScanRDD - .mapPartitionsInternal(batches => batches.map(batch => batch.numRows().toLong)) - .collect() - .sum - assert(chRowCnt == rowCnt, "The row count of the benchmark is not equal.") - - parquetReadBenchmark.addCase(s"Vanilla Spark Parquet Read", executedCnt) { - _ => - val resultRDD: RDD[Long] = newFileScanRDD.mapPartitionsInternal { - batches => batches.map(_.numRows().toLong) - } - resultRDD.collect() - } - - parquetReadBenchmark.addCase(s"Vanilla Spark Parquet Read to Rows", executedCnt) { - _ => - val resultRDD: RDD[Long] = newFileScanRDD.mapPartitionsInternal { - batches => - val toUnsafe = UnsafeProjection.create(fileScanOutput, fileScanOutput) - batches.map { - batch => - // Convert to row and decode parquet value - batch.rowIterator().asScala.map(toUnsafe).foreach(_.numFields) - batch.numRows().toLong - } - } - resultRDD.collect() - } - } - - parquetReadBenchmark.run() - } -} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index f1fbf3648bb2..5f9b5afa9976 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -27,9 +27,8 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.utils._ import org.apache.gluten.vectorized._ -import org.apache.spark.{SparkConf, SparkContext, TaskContext} +import org.apache.spark.{SparkConf, TaskContext} import org.apache.spark.internal.Logging -import org.apache.spark.rdd.RDD import org.apache.spark.softaffinity.SoftAffinity import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter} @@ -232,16 +231,4 @@ class VeloxIteratorApi extends IteratorApi with Logging { .create() } // scalastyle:on argcount - - /** Generate Native FileScanRDD, currently only for ClickHouse Backend. */ - override def genNativeFileScanRDD( - sparkContext: SparkContext, - wsCxt: WholeStageTransformContext, - splitInfos: Seq[SplitInfo], - scan: BasicScanExecTransformer, - numOutputRows: SQLMetric, - numOutputBatches: SQLMetric, - scanTime: SQLMetric): RDD[ColumnarBatch] = { - throw new UnsupportedOperationException("Cannot support to generate Native FileScanRDD.") - } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala index d999948d7047..53dc8f47861f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala @@ -24,7 +24,6 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.substrait.rel.SplitInfo import org.apache.spark._ -import org.apache.spark.rdd.RDD import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.StructType @@ -81,14 +80,4 @@ trait IteratorApi { partitionIndex: Int, materializeInput: Boolean = false): Iterator[ColumnarBatch] // scalastyle:on argcount - - /** Generate Native FileScanRDD, currently only for ClickHouse Backend. */ - def genNativeFileScanRDD( - sparkContext: SparkContext, - wsCxt: WholeStageTransformContext, - splitInfos: Seq[SplitInfo], - scan: BasicScanExecTransformer, - numOutputRows: SQLMetric, - numOutputBatches: SQLMetric, - scanTime: SQLMetric): RDD[ColumnarBatch] } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index 2dd5aff766a9..af35957ec393 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -22,18 +22,14 @@ import org.apache.gluten.extension.ValidationResult import org.apache.gluten.substrait.`type`.ColumnTypeNode import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.extensions.ExtensionBuilder -import org.apache.gluten.substrait.plan.PlanBuilder import org.apache.gluten.substrait.rel.{RelBuilder, SplitInfo} import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat -import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.hive.HiveTableScanExecTransformer import org.apache.spark.sql.types.{BooleanType, StringType, StructField, StructType} -import org.apache.spark.sql.vectorized.ColumnarBatch -import com.google.common.collect.Lists import com.google.protobuf.StringValue import scala.collection.JavaConverters._ @@ -75,28 +71,6 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource .genSplitInfo(_, getPartitionSchema, fileFormat, getMetadataColumns.map(_.name))) } - def doExecuteColumnarInternal(): RDD[ColumnarBatch] = { - val numOutputRows = longMetric("numOutputRows") - val numOutputVectors = longMetric("outputVectors") - val scanTime = longMetric("scanTime") - val substraitContext = new SubstraitContext - val transformContext = transform(substraitContext) - val outNames = - filteRedundantField(outputAttributes()).map(ConverterUtils.genColumnNameWithExprId).asJava - val planNode = - PlanBuilder.makePlan(substraitContext, Lists.newArrayList(transformContext.root), outNames) - - BackendsApiManager.getIteratorApiInstance.genNativeFileScanRDD( - sparkContext, - WholeStageTransformContext(planNode, substraitContext), - getSplitInfos, - this, - numOutputRows, - numOutputVectors, - scanTime - ) - } - override protected def doValidateInternal(): ValidationResult = { var fields = schema.fields @@ -182,9 +156,9 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource def filteRedundantField(outputs: Seq[Attribute]): Seq[Attribute] = { var final_output: List[Attribute] = List() val outputList = outputs.toArray - for (i <- 0 to outputList.size - 1) { + for (i <- outputList.indices) { var dup = false - for (j <- 0 to i - 1) { + for (j <- 0 until i) { if (outputList(i).name == outputList(j).name) { dup = true } @@ -193,6 +167,6 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource final_output = final_output :+ outputList(i) } } - final_output.toSeq + final_output } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala index b0c8c59e7bb5..64d9d6546bd8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala @@ -22,7 +22,6 @@ import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat -import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.QueryPlan @@ -31,7 +30,6 @@ import org.apache.spark.sql.connector.read.{InputPartition, Scan} import org.apache.spark.sql.execution.datasources.v2.{BatchScanExecShim, FileScan} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.vectorized.ColumnarBatch /** Columnar Based BatchScanExec. */ case class BatchScanExecTransformer( @@ -144,10 +142,6 @@ abstract class BatchScanExecTransformerBase( super.doValidateInternal() } - override def doExecuteColumnar(): RDD[ColumnarBatch] = { - doExecuteColumnarInternal() - } - override def metricsUpdater(): MetricsUpdater = BackendsApiManager.getMetricsApiInstance.genBatchScanTransformerMetricsUpdater(metrics) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala index c3d2da7f0466..ff905251b8ae 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala @@ -21,7 +21,6 @@ import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat -import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, PlanExpression} import org.apache.spark.sql.catalyst.plans.QueryPlan @@ -30,7 +29,6 @@ import org.apache.spark.sql.execution.FileSourceScanExecShim import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.collection.BitSet case class FileSourceScanExecTransformer( @@ -147,10 +145,6 @@ abstract class FileSourceScanExecTransformerBase( super.doValidateInternal() } - override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { - doExecuteColumnarInternal() - } - override def metricsUpdater(): MetricsUpdater = BackendsApiManager.getMetricsApiInstance.genFileSourceScanTransformerMetricsUpdater(metrics) diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala index 2952267e5a1d..5dfa85b269a8 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala @@ -22,7 +22,6 @@ import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat -import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSeq, Expression} @@ -34,7 +33,6 @@ import org.apache.spark.sql.hive.HiveTableScanExecTransformer._ import org.apache.spark.sql.hive.client.HiveClientImpl import org.apache.spark.sql.hive.execution.{AbstractHiveTableScanExec, HiveTableScanExec} import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.Utils import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat @@ -81,10 +79,6 @@ case class HiveTableScanExecTransformer( override def metricsUpdater(): MetricsUpdater = BackendsApiManager.getMetricsApiInstance.genHiveTableScanTransformerMetricsUpdater(metrics) - override def doExecuteColumnar(): RDD[ColumnarBatch] = { - doExecuteColumnarInternal() - } - @transient private lazy val hivePartitionConverter = new HivePartitionConverter(session.sessionState.newHadoopConf(), session) From 2c89fb1814f429ed22f0cce6a5bd68a15e00e6f6 Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Fri, 31 May 2024 18:08:54 +0800 Subject: [PATCH 190/402] [GLUTEN-5901][CH] Support CH backend parquet + delta (#5902) Support CH backend to read/write parquet with the delta: 1. native read parquet from the delta catalog; 2. fallback write the parquet to the delta catalog ( don't support the DeltaInvariantCheckerExec operator and DeltaTaskStatisticsTracker) ; 3. Use the ClickHouseSparkCatalog as the uniform catalog. Close #5901. --- backends-clickhouse/pom.xml | 6 + .../org/apache/spark/sql/delta/DeltaLog.scala | 30 +- .../org/apache/spark/sql/delta/Snapshot.scala | 55 +- .../sql/delta/commands/DeleteCommand.scala | 2 + .../sql/delta/commands/MergeIntoCommand.scala | 2 + .../delta/commands/OptimizeTableCommand.scala | 67 +- .../sql/delta/commands/UpdateCommand.scala | 2 + .../sql/delta/commands/VacuumCommand.scala | 134 +- .../delta/files/MergeTreeCommitProtocol.scala | 16 +- .../sql/delta/commands/VacuumCommand.scala | 464 ------ .../io/delta/tables/ClickhouseTable.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 617 ++++--- .../org/apache/spark/sql/delta/Snapshot.scala | 434 +++-- .../sql/delta/commands/DeleteCommand.scala | 368 +++-- .../sql/delta/commands/MergeIntoCommand.scala | 643 ++++---- .../delta/commands/OptimizeTableCommand.scala | 392 +++-- .../sql/delta/commands/UpdateCommand.scala | 258 ++- .../sql/delta/commands/VacuumCommand.scala | 575 +++++++ .../delta/files/MergeTreeCommitProtocol.scala | 138 +- .../v2/clickhouse/DeltaLogAdapter.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 6 +- .../clickhouse/CHTransformerApi.scala | 2 - .../ClickhouseOptimisticTransaction.scala | 283 ++-- .../sql/delta/catalog/ClickHouseTableV2.scala | 4 +- .../OptimizeTableCommandOverwrites.scala | 28 +- .../v2/clickhouse/ClickHouseConfig.scala | 18 +- .../clickhouse/ClickHouseSparkCatalog.scala | 269 +++- ...utenClickHouseDeltaParquetWriteSuite.scala | 1430 +++++++++++++++++ ...utenClickHouseMergeTreeOptimizeSuite.scala | 93 +- .../FileSourceScanExecTransformer.scala | 1 + gluten-ut/pom.xml | 2 +- pom.xml | 10 +- 32 files changed, 4241 insertions(+), 2108 deletions(-) delete mode 100644 backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename backends-clickhouse/src/main/{delta-22 => delta-23}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/DeltaLog.scala (65%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/Snapshot.scala (59%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (61%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala (70%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (51%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (66%) create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala (68%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala (100%) rename backends-clickhouse/src/main/{delta-22 => delta-23}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (94%) create mode 100644 backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 94df2f36f183..a37734b150bb 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -270,6 +270,12 @@ src/main/delta-${delta.binary.version}/**/*.scala src/test/delta-${delta.binary.version}/**/*.scala + + src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/*.scala + src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/files/*.scala + src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/DeltaLog.scala + src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/Snapshot.scala + diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala index 0f6455997e56..57c6c8550fc0 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStoreProvider +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} import org.apache.spark.sql.types.{StructField, StructType} @@ -214,7 +215,9 @@ class DeltaLog private ( */ def startTransaction(): OptimisticTransaction = { update() + // --- modified start new ClickhouseOptimisticTransaction(this, None) + // --- modified end } /** @@ -443,7 +446,13 @@ class DeltaLog private ( val fileIndex = TahoeLogFileIndex(spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) - val bucketSpec: Option[BucketSpec] = ClickHouseTableV2.getTable(this).bucketOption + // --- modified start + val bucketSpec: Option[BucketSpec] = + if (ClickHouseConfig.isMergeTreeFormatEngine(snapshotToUse.metadata.configuration)) { + ClickHouseTableV2.getTable(this).bucketOption + } else { + None + } new DeltaHadoopFsRelation( fileIndex, partitionSchema = @@ -464,20 +473,28 @@ class DeltaLog private ( spark, this ) + // --- modified end } - override def fileFormat(metadata: Metadata = metadata): FileFormat = - ClickHouseTableV2.getTable(this).getFileFormat(metadata) - + override def fileFormat(metadata: Metadata = metadata): FileFormat = { + // --- modified start + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + ClickHouseTableV2.getTable(this).getFileFormat(metadata) + } else { + super.fileFormat(metadata) + } + // --- modified end + } } object DeltaLog extends DeltaLogging { + // --- modified start @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) private class DeltaHadoopFsRelation( location: FileIndex, partitionSchema: StructType, - // The top-level columns in `dataSchema` should match the actual physical file schema, otherwise - // the ORC data source may not work with the by-ordinal mode. + // The top-level columns in `dataSchema` should match the actual physical file schema, + // otherwise the ORC data source may not work with the by-ordinal mode. dataSchema: StructType, bucketSpec: Option[BucketSpec], fileFormat: FileFormat, @@ -502,6 +519,7 @@ object DeltaLog extends DeltaLogging { ).run(sparkSession) } } + // --- modified end /** * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala index 2233aa0cd1ef..2e4d6bb2207c 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DataSkippingReader, DeltaScan, FileSizeHistogram, StatisticsCollection} import org.apache.spark.sql.delta.util.StateCache import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.{SerializableConfiguration, Utils} @@ -404,6 +405,7 @@ class Snapshot( s"${getClass.getSimpleName}(path=$path, version=$version, metadata=$metadata, " + s"logSegment=$logSegment, checksumOpt=$checksumOpt)" + // --- modified start override def filesForScan( projection: Seq[Attribute], filters: Seq[Expression], @@ -418,31 +420,36 @@ class Snapshot( } private def replaceWithAddMergeTreeParts(deltaScan: DeltaScan) = { - DeltaScan.apply( - deltaScan.version, - deltaScan.files - .map( - addFile => { - val addFileAsKey = AddFileAsKey(addFile) - - val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) - // this is for later use - ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) - ret - }), - deltaScan.total, - deltaScan.partition, - deltaScan.scanned - )( - deltaScan.scannedSnapshot, - deltaScan.partitionFilters, - deltaScan.dataFilters, - deltaScan.unusedFilters, - deltaScan.projection, - deltaScan.scanDurationMs, - deltaScan.dataSkippingType - ) + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + DeltaScan.apply( + deltaScan.version, + deltaScan.files + .map( + addFile => { + val addFileAsKey = AddFileAsKey(addFile) + + val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) + // this is for later use + ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) + ret + }), + deltaScan.total, + deltaScan.partition, + deltaScan.scanned + )( + deltaScan.scannedSnapshot, + deltaScan.partitionFilters, + deltaScan.dataFilters, + deltaScan.unusedFilters, + deltaScan.projection, + deltaScan.scanDurationMs, + deltaScan.dataSkippingType + ) + } else { + deltaScan + } } + // --- modified end logInfo(s"Created snapshot $this") init() diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala index 527b9619eb5d..61b0330723f5 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala @@ -216,6 +216,7 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt if (candidateFiles.isEmpty) { Array.empty[String] } else { + // --- modified start data .filter(new Column(cond)) .select(input_file_name().as("input_files")) @@ -224,6 +225,7 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt .distinct() .as[String] .collect() + // --- modified end } } diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala index 89208dd45314..4b26d5d5d949 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala @@ -407,7 +407,9 @@ case class MergeIntoCommand( val recordTouchedFileName = udf { (fileName: String) => { + // --- modified start fileName.split(",").foreach(name => touchedFilesAccum.add(name)) + // --- modified end 1 } }.asNondeterministic() diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index 571ac63f1eb6..6437ab75903b 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.skipping.MultiDimClustering import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils import org.apache.spark.sql.execution.metric.SQLMetric @@ -127,8 +128,10 @@ case class OptimizeTableCommand( override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil override def run(sparkSession: SparkSession): Seq[Row] = { + // --- modified start CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession) val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE") + // --- modified end val partitionColumns = deltaLog.snapshot.metadata.partitionColumns // Parse the predicate expression into Catalyst expression and verify only simple filters @@ -177,6 +180,10 @@ class OptimizeExecutor( def optimize(): Seq[Row] = { recordDeltaOperation(deltaLog, "delta.optimize") { + // --- modified start + val isMergeTreeFormat = ClickHouseConfig + .isMergeTreeFormatEngine(deltaLog.snapshot.metadata.configuration) + // --- modified end val minFileSize = sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE) val maxFileSize = @@ -194,37 +201,59 @@ class OptimizeExecutor( // select all files in case of multi-dimensional clustering val filesToProcess = candidateFiles.filter(_.size < minFileSize || isMultiDimClustering) - val partitionsToCompact = filesToProcess - .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) - .toSeq - - val jobs = groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) - - val parallelJobCollection = new ParVector(jobs.toVector) + // --- modified start // Create a task pool to parallelize the submission of optimization jobs to Spark. val threadPool = ThreadUtils.newForkJoinPool( "OptimizeJob", sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_THREADS)) + val (updates, jobs) = if (isMergeTreeFormat) { + val partitionsToCompact = filesToProcess + .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) + .toSeq + + val jobs = groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) + + val parallelJobCollection = new ParVector(jobs.toVector) + + val updates = + try { + val forkJoinPoolTaskSupport = new ForkJoinTaskSupport(threadPool) + parallelJobCollection.tasksupport = forkJoinPoolTaskSupport + + parallelJobCollection + .flatMap( + partitionBinGroup => + runOptimizeBinJobClickhouse( + txn, + partitionBinGroup._1._2, + partitionBinGroup._1._1, + partitionBinGroup._2, + maxFileSize)) + .seq + } finally { + threadPool.shutdownNow() + } + (updates, jobs) + } else { + val partitionsToCompact = filesToProcess.groupBy(_.partitionValues).toSeq + + val jobs = groupFilesIntoBins(partitionsToCompact, maxFileSize) - val updates = - try { + val parallelJobCollection = new ParVector(jobs.toVector) + + val updates = try { val forkJoinPoolTaskSupport = new ForkJoinTaskSupport(threadPool) parallelJobCollection.tasksupport = forkJoinPoolTaskSupport - parallelJobCollection - .flatMap( - partitionBinGroup => - runOptimizeBinJobClickhouse( - txn, - partitionBinGroup._1._2, - partitionBinGroup._1._1, - partitionBinGroup._2, - maxFileSize)) - .seq + parallelJobCollection.flatMap(partitionBinGroup => + runOptimizeBinJob(txn, partitionBinGroup._1, partitionBinGroup._2, maxFileSize)).seq } finally { threadPool.shutdownNow() } + (updates, jobs) + } + // --- modified end val addedFiles = updates.collect { case a: AddFile => a } val removedFiles = updates.collect { case r: RemoveFile => r } diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala index f6e2968b703f..42a081788eb1 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala @@ -144,6 +144,7 @@ case class UpdateCommand( }.asNondeterministic() val pathsToRewrite = withStatusCode("DELTA", UpdateCommand.FINDING_TOUCHED_FILES_MSG) { + // --- modified start data .filter(new Column(updateCondition)) .filter(updatedRowUdf()) @@ -152,6 +153,7 @@ case class UpdateCommand( .distinct() .as[String] .collect() + // --- modified end } scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 11f6aa977904..3a390f64d559 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaFileOperations +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.functions.{col, expr, when} import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} @@ -41,7 +42,9 @@ import java.util.concurrent.TimeUnit */ object VacuumCommand extends VacuumCommandImpl with Serializable { - case class FileNameAndSize(path: String, length: Long, isDir: Boolean) + // --- modified start + case class FileNameAndSize(path: String, length: Long, isDir: Boolean = false) + // --- modified end /** * Additional check on retention duration to prevent people from shooting themselves in the foot. @@ -112,6 +115,11 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { "No state defined for this table. Is this really " + "a Delta table? Refusing to garbage collect.") + // --- modified start + val isMergeTreeFormat = ClickHouseConfig + .isMergeTreeFormatEngine(deltaLog.snapshot.metadata.configuration) + // --- modified end + val retentionMillis = retentionHours.map(h => TimeUnit.HOURS.toMillis(math.round(h))) checkRetentionPeriodSafety(spark, retentionMillis, deltaLog.tombstoneRetentionMillis) @@ -209,60 +217,92 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { // 5. We subtract all the valid files and tombstones in our state // 6. We filter all paths with a count of 1, which will correspond to files not in the // state, and empty directories. We can safely delete all of these - val diff_temp = allFilesAndDirs - .where('modificationTime < deleteBeforeTimestamp || 'isDir) - .mapPartitions { - fileStatusIterator => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - fileStatusIterator.flatMap { - fileStatus => - if (fileStatus.isDir) { - implicit val fileNameAndSizeEncoder = - org.apache.spark.sql.Encoders.product[FileNameAndSize] - Iterator.single( - FileNameAndSize( - relativize(fileStatus.getPath, fs, reservoirBase, isDir = true), - 0, - true) - ) - } else { - val dirs = getAllSubdirs(basePath, fileStatus.path, fs) - val dirsWithSlash = dirs.map { - p => + // --- modified start + val diff = if (isMergeTreeFormat) { + val diff_temp = allFilesAndDirs + .where('modificationTime < deleteBeforeTimestamp || 'isDir) + .mapPartitions { + fileStatusIterator => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + fileStatusIterator.flatMap { + fileStatus => + if (fileStatus.isDir) { + implicit val fileNameAndSizeEncoder = + org.apache.spark.sql.Encoders.product[FileNameAndSize] + Iterator.single( FileNameAndSize( - relativize(new Path(p), fs, reservoirBase, isDir = true), + relativize(fileStatus.getPath, fs, reservoirBase, isDir = true), 0, true) + ) + } else { + val dirs = getAllSubdirs(basePath, fileStatus.path, fs) + val dirsWithSlash = dirs.map { + p => + FileNameAndSize( + relativize(new Path(p), fs, reservoirBase, isDir = true), + 0, + true) + } + dirsWithSlash ++ Iterator( + FileNameAndSize( + relativize(new Path(fileStatus.path), fs, reservoirBase, isDir = false), + 0, + false)) } - dirsWithSlash ++ Iterator( - FileNameAndSize( - relativize(new Path(fileStatus.path), fs, reservoirBase, isDir = false), - 0, - false)) + } + } + .withColumn( + "dir", + when(col("isDir"), col("path")) + .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) + .groupBy(col("path"), col("dir")) + .count() + + diff_temp + .join(validFiles, diff_temp("dir") === validFiles("path"), "leftanti") + .where('count === 1) + .select('path) + .as[String] + .map { + relativePath => + assert( + !stringToPath(relativePath).isAbsolute, + "Shouldn't have any absolute paths for deletion here.") + pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) + } + } else { + allFilesAndDirs + .where('modificationTime < deleteBeforeTimestamp || 'isDir) + .mapPartitions { fileStatusIterator => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + fileStatusIterator.flatMap { fileStatus => + if (fileStatus.isDir) { + Iterator.single(relativize(fileStatus.getPath, fs, reservoirBase, isDir = true)) + } else { + val dirs = getAllSubdirs(basePath, fileStatus.path, fs) + val dirsWithSlash = dirs.map { p => + relativize(new Path(p), fs, reservoirBase, isDir = true) } + dirsWithSlash ++ Iterator( + relativize(new Path(fileStatus.path), fs, reservoirBase, isDir = false)) + } } - } -// .groupBy(col("path")) - .withColumn( - "dir", - when(col("isDir"), col("path")) - .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) - .groupBy(col("path"), col("dir")) - .count() - - val diff = diff_temp - .join(validFiles, diff_temp("dir") === validFiles("path"), "leftanti") - .where('count === 1) - .select('path) - .as[String] - .map { - relativePath => - assert( - !stringToPath(relativePath).isAbsolute, + }.groupBy($"value" as 'path) + .count() + .join(validFiles, Seq("path"), "leftanti") + .where('count === 1) + .select('path) + .as[String] + .map { relativePath => + assert(!stringToPath(relativePath).isAbsolute, "Shouldn't have any absolute paths for deletion here.") pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) - } + } + } + // --- modified end if (dryRun) { val numFiles = diff.count() diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala index bfc2555e82a2..4b6d56644b9e 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala @@ -212,19 +212,9 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O } override def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage = { - if (addedFiles.nonEmpty) { - /* val fs = new Path(path, addedFiles.head._2).getFileSystem(taskContext.getConfiguration) - val statuses: Seq[FileAction] = addedFiles.map { f => - val filePath = new Path(path, new Path(new URI(f._2))) - val stat = fs.getFileStatus(filePath) - - buildActionFromAddedFile(f, stat, taskContext) - }.toSeq */ - - new TaskCommitMessage(Nil) - } else { - new TaskCommitMessage(Nil) - } + // will return TaskCommitMessage(Nil) directly, + // the FileStatus list will be get from the CH backend. + new TaskCommitMessage(Nil) } override def abortTask(taskContext: TaskAttemptContext): Unit = { diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/VacuumCommand.scala deleted file mode 100644 index dc833c2b2fae..000000000000 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.spark.broadcast.Broadcast -import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.DeltaFileOperations -import org.apache.spark.sql.delta.util.DeltaFileOperations.tryDeleteNonRecursive -import org.apache.spark.sql.functions._ -import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} - -// scalastyle:off import.ordering.noEmptyLine -import com.fasterxml.jackson.databind.annotation.JsonDeserialize -import org.apache.hadoop.fs.{FileSystem, Path} - -import java.net.URI -import java.util.Date -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: - * 1. In Gluten, part is a directory, but VacuumCommand assumes part is a file. So we need some - * modifications to make it work. - */ - -/** - * Vacuums the table by clearing all untracked files and folders within this table. First lists all - * the files and directories in the table, and gets the relative paths with respect to the base of - * the table. Then it gets the list of all tracked files for this table, which may or may not be - * within the table base path, and gets the relative paths of all the tracked files with respect to - * the base of the table. Files outside of the table path will be ignored. Then we take a diff of - * the files and delete directories that were already empty, and all files that are within the table - * that are no longer tracked. - */ -object VacuumCommand extends VacuumCommandImpl with Serializable { - - case class FileNameAndSize(path: String, length: Long, isDir: Boolean) - - /** - * Additional check on retention duration to prevent people from shooting themselves in the foot. - */ - protected def checkRetentionPeriodSafety( - spark: SparkSession, - retentionMs: Option[Long], - configuredRetention: Long): Unit = { - require(retentionMs.forall(_ >= 0), "Retention for Vacuum can't be less than 0.") - val checkEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) - val retentionSafe = retentionMs.forall(_ >= configuredRetention) - var configuredRetentionHours = TimeUnit.MILLISECONDS.toHours(configuredRetention) - if (TimeUnit.HOURS.toMillis(configuredRetentionHours) < configuredRetention) { - configuredRetentionHours += 1 - } - require( - !checkEnabled || retentionSafe, - s"""Are you sure you would like to vacuum files with such a low retention period? If you have - |writers that are currently writing to this table, there is a risk that you may corrupt the - |state of your Delta table. - | - |If you are certain that there are no operations being performed on this table, such as - |insert/upsert/delete/optimize, then you may turn off this check by setting: - |spark.databricks.delta.retentionDurationCheck.enabled = false - | - |If you are not sure, please use a value not less than "$configuredRetentionHours hours". - """.stripMargin - ) - } - - /** - * Clears all untracked files and folders within this table. First lists all the files and - * directories in the table, and gets the relative paths with respect to the base of the table. - * Then it gets the list of all tracked files for this table, which may or may not be within the - * table base path, and gets the relative paths of all the tracked files with respect to the base - * of the table. Files outside of the table path will be ignored. Then we take a diff of the files - * and delete directories that were already empty, and all files that are within the table that - * are no longer tracked. - * - * @param dryRun - * If set to true, no files will be deleted. Instead, we will list all files and directories - * that will be cleared. - * @param retentionHours - * An optional parameter to override the default Delta tombstone retention period - * @return - * A Dataset containing the paths of the files/folders to delete in dryRun mode. Otherwise - * returns the base path of the table. - */ - def gc( - spark: SparkSession, - deltaLog: DeltaLog, - dryRun: Boolean = true, - retentionHours: Option[Double] = None, - clock: Clock = new SystemClock): DataFrame = { - recordDeltaOperation(deltaLog, "delta.gc") { - - val path = deltaLog.dataPath - val deltaHadoopConf = deltaLog.newDeltaHadoopConf() - val fs = path.getFileSystem(deltaHadoopConf) - - import org.apache.spark.sql.delta.implicits._ - - val snapshot = deltaLog.update() - - require( - snapshot.version >= 0, - "No state defined for this table. Is this really " + - "a Delta table? Refusing to garbage collect.") - - val retentionMillis = retentionHours.map(h => TimeUnit.HOURS.toMillis(math.round(h))) - checkRetentionPeriodSafety(spark, retentionMillis, deltaLog.tombstoneRetentionMillis) - - val deleteBeforeTimestamp = retentionMillis - .map(millis => clock.getTimeMillis() - millis) - .getOrElse(deltaLog.minFileRetentionTimestamp) - logInfo( - s"Starting garbage collection (dryRun = $dryRun) of untracked files older than " + - s"${new Date(deleteBeforeTimestamp).toString} in $path") - val hadoopConf = spark.sparkContext.broadcast(new SerializableConfiguration(deltaHadoopConf)) - val basePath = fs.makeQualified(path).toString - var isBloomFiltered = false - val parallelDeleteEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_ENABLED) - val parallelDeletePartitions = - spark.sessionState.conf - .getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM) - .getOrElse(spark.sessionState.conf.numShufflePartitions) - val relativizeIgnoreError = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) - val startTimeToIdentifyEligibleFiles = System.currentTimeMillis() - val validFiles = snapshot.stateDS - .mapPartitions { - actions => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - actions.flatMap { - _.unwrap match { - case tombstone: RemoveFile if tombstone.delTimestamp < deleteBeforeTimestamp => - Nil - case fa: FileAction => - getValidRelativePathsAndSubdirs( - fa, - fs, - reservoirBase, - relativizeIgnoreError, - isBloomFiltered) - case _ => Nil - } - } - } - .toDF("path") - - val partitionColumns = snapshot.metadata.partitionSchema.fieldNames - val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism - - val allFilesAndDirs = DeltaFileOperations - .recursiveListDirs( - spark, - Seq(basePath), - hadoopConf, - hiddenDirNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), - hiddenFileNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), - fileListingParallelism = Option(parallelism) - ) - .groupByKey(_.path) - .mapGroups { - (k, v) => - val duplicates = v.toSeq - // of all the duplicates we can return the newest file. - duplicates.maxBy(_.modificationTime) - } - - try { - allFilesAndDirs.cache() - - implicit val fileNameAndSizeEncoder = org.apache.spark.sql.Encoders.product[FileNameAndSize] - - val dirCounts = allFilesAndDirs.where(col("isDir")).count() + 1 // +1 for the base path - - // The logic below is as follows: - // 1. We take all the files and directories listed in our reservoir - // 2. We filter all files older than our tombstone retention period and directories - // 3. We get the subdirectories of all files so that we can find non-empty directories - // 4. We groupBy each path, and count to get how many files are in each sub-directory - // 5. We subtract all the valid files and tombstones in our state - // 6. We filter all paths with a count of 1, which will correspond to files not in the - // state, and empty directories. We can safely delete all of these - val diff_temp = allFilesAndDirs - .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) - .mapPartitions { - fileStatusIterator => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - fileStatusIterator.flatMap { - fileStatus => - if (fileStatus.isDir) { - Iterator.single( - FileNameAndSize( - relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), - 0L, - true)) - } else { - val dirs = getAllSubdirs(basePath, fileStatus.path, fs) - val dirsWithSlash = dirs.map { - p => - val relativizedPath = - relativize(new Path(p), fs, reservoirBase, isDir = true) - FileNameAndSize(relativizedPath, 0L, true) - } - dirsWithSlash ++ Iterator( - FileNameAndSize( - relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), - fileStatus.length, - false)) - } - } - } - .withColumn( - "dir", - when(col("isDir"), col("path")) - .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) - .groupBy(col("path"), col("dir")) - .agg(count(new Column("*")).as("count"), sum("length").as("length")) - - val diff = diff_temp - .join(validFiles, diff_temp("dir") === validFiles("path"), "leftanti") - .where(col("count") === 1) - - val sizeOfDataToDeleteRow = diff.agg(sum("length").cast("long")).first - val sizeOfDataToDelete = if (sizeOfDataToDeleteRow.isNullAt(0)) { - 0L - } else { - sizeOfDataToDeleteRow.getLong(0) - } - - val diffFiles = diff - .select(col("path")) - .as[String] - .map { - relativePath => - assert( - !stringToPath(relativePath).isAbsolute, - "Shouldn't have any absolute paths for deletion here.") - pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) - } - val timeTakenToIdentifyEligibleFiles = - System.currentTimeMillis() - startTimeToIdentifyEligibleFiles - - if (dryRun) { - val numFiles = diffFiles.count() - val stats = DeltaVacuumStats( - isDryRun = true, - specifiedRetentionMillis = retentionMillis, - defaultRetentionMillis = deltaLog.tombstoneRetentionMillis, - minRetainedTimestamp = deleteBeforeTimestamp, - dirsPresentBeforeDelete = dirCounts, - objectsDeleted = numFiles, - sizeOfDataToDelete = sizeOfDataToDelete, - timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, - timeTakenForDelete = 0L - ) - - recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) - logConsole( - s"Found $numFiles files and directories in a total of " + - s"$dirCounts directories that are safe to delete.$stats") - - return diffFiles.map(f => stringToPath(f).toString).toDF("path") - } - logVacuumStart( - spark, - deltaLog, - path, - diffFiles, - sizeOfDataToDelete, - retentionMillis, - deltaLog.tombstoneRetentionMillis) - - val deleteStartTime = System.currentTimeMillis() - val filesDeleted = - try { - delete( - diffFiles, - spark, - basePath, - hadoopConf, - parallelDeleteEnabled, - parallelDeletePartitions) - } catch { - case t: Throwable => - logVacuumEnd(deltaLog, spark, path) - throw t - } - val timeTakenForDelete = System.currentTimeMillis() - deleteStartTime - val stats = DeltaVacuumStats( - isDryRun = false, - specifiedRetentionMillis = retentionMillis, - defaultRetentionMillis = deltaLog.tombstoneRetentionMillis, - minRetainedTimestamp = deleteBeforeTimestamp, - dirsPresentBeforeDelete = dirCounts, - objectsDeleted = filesDeleted, - sizeOfDataToDelete = sizeOfDataToDelete, - timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, - timeTakenForDelete = timeTakenForDelete - ) - recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) - logVacuumEnd(deltaLog, spark, path, Some(filesDeleted), Some(dirCounts)) - - spark.createDataset(Seq(basePath)).toDF("path") - } finally { - allFilesAndDirs.unpersist() - } - } - } -} - -trait VacuumCommandImpl extends DeltaCommand { - - protected def logVacuumStart( - spark: SparkSession, - deltaLog: DeltaLog, - path: Path, - diff: Dataset[String], - sizeOfDataToDelete: Long, - specifiedRetentionMillis: Option[Long], - defaultRetentionMillis: Long): Unit = { - logInfo( - s"Deleting untracked files and empty directories in $path. The amount of data to be " + - s"deleted is $sizeOfDataToDelete (in bytes)") - } - - protected def logVacuumEnd( - deltaLog: DeltaLog, - spark: SparkSession, - path: Path, - filesDeleted: Option[Long] = None, - dirCounts: Option[Long] = None): Unit = { - if (filesDeleted.nonEmpty) { - logConsole( - s"Deleted ${filesDeleted.get} files and directories in a total " + - s"of ${dirCounts.get} directories.") - } - } - - /** - * Attempts to relativize the `path` with respect to the `reservoirBase` and converts the path to - * a string. - */ - protected def relativize( - path: Path, - fs: FileSystem, - reservoirBase: Path, - isDir: Boolean): String = { - pathToString(DeltaFileOperations.tryRelativizePath(fs, reservoirBase, path)) - } - - /** - * Wrapper function for DeltaFileOperations.getAllSubDirectories returns all subdirectories that - * `file` has with respect to `base`. - */ - protected def getAllSubdirs(base: String, file: String, fs: FileSystem): Iterator[String] = { - DeltaFileOperations.getAllSubDirectories(base, file)._1 - } - - /** Attempts to delete the list of candidate files. Returns the number of files deleted. */ - protected def delete( - diff: Dataset[String], - spark: SparkSession, - basePath: String, - hadoopConf: Broadcast[SerializableConfiguration], - parallel: Boolean, - parallelPartitions: Int): Long = { - import org.apache.spark.sql.delta.implicits._ - - if (parallel) { - diff - .repartition(parallelPartitions) - .mapPartitions { - files => - val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) - val filesDeletedPerPartition = - files.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) - Iterator(filesDeletedPerPartition) - } - .collect() - .sum - } else { - val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) - val fileResultSet = diff.toLocalIterator().asScala - fileResultSet.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) - } - } - - protected def stringToPath(path: String): Path = new Path(new URI(path)) - - protected def pathToString(path: Path): String = path.toUri.toString - - /** Returns the relative path of a file action or None if the file lives outside of the table. */ - protected def getActionRelativePath( - action: FileAction, - fs: FileSystem, - basePath: Path, - relativizeIgnoreError: Boolean): Option[String] = { - val filePath = stringToPath(action.path) - if (filePath.isAbsolute) { - val maybeRelative = - DeltaFileOperations.tryRelativizePath(fs, basePath, filePath, relativizeIgnoreError) - if (maybeRelative.isAbsolute) { - // This file lives outside the directory of the table. - None - } else { - Some(pathToString(maybeRelative)) - } - } else { - Some(pathToString(filePath)) - } - } - - /** - * Returns the relative paths of all files and subdirectories for this action that must be - * retained during GC. - */ - protected def getValidRelativePathsAndSubdirs( - action: FileAction, - fs: FileSystem, - basePath: Path, - relativizeIgnoreError: Boolean, - isBloomFiltered: Boolean): Seq[String] = { - getActionRelativePath(action, fs, basePath, relativizeIgnoreError) - .map(relativePath => Seq(relativePath) ++ getAllSubdirs("/", relativePath, fs)) - .getOrElse(Seq.empty) - } -} - -case class DeltaVacuumStats( - isDryRun: Boolean, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - specifiedRetentionMillis: Option[Long], - defaultRetentionMillis: Long, - minRetainedTimestamp: Long, - dirsPresentBeforeDelete: Long, - objectsDeleted: Long, - sizeOfDataToDelete: Long, - timeTakenToIdentifyEligibleFiles: Long, - timeTakenForDelete: Long) diff --git a/backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src/main/delta-23/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-22/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src/main/delta-23/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala similarity index 65% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala index 4cab6454d15a..78fbc3fcdb99 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala @@ -14,15 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} -import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expression, Literal} -import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper -import org.apache.spark.sql.catalyst.util.FailFastMode +// scalastyle:off import.ordering.noEmptyLine +import java.io.File +import java.lang.ref.WeakReference +import java.net.URI +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantLock + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.Try +import scala.util.control.NonFatal + +import com.databricks.spark.util.TagDefinitions._ import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.delta.commands.WriteIntoDelta @@ -30,9 +37,21 @@ import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeLogFileIndex} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} -import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.sources._ import org.apache.spark.sql.delta.storage.LogStoreProvider +import com.google.common.cache.{CacheBuilder, RemovalNotification} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} +import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expression, Literal} +import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper +import org.apache.spark.sql.catalyst.util.FailFastMode import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} @@ -40,56 +59,46 @@ import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util._ -// scalastyle:off import.ordering.noEmptyLine -import com.databricks.spark.util.TagDefinitions._ -import com.google.common.cache.{CacheBuilder, RemovalNotification} -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} - -import java.io.File -import java.lang.ref.WeakReference -import java.net.URI -import java.util.concurrent.TimeUnit -import java.util.concurrent.locks.ReentrantLock - -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.util.Try -import scala.util.control.NonFatal - -// This class is copied from Delta 2.2.0 because it has a private constructor, -// which makes it impossible to extend - /** * Gluten overwrite Delta: * - * This file is copied from Delta 2.2.0 It is modified to overcome the following issues: - * 1. return ClickhouseOptimisticTransaction 2. return DeltaMergeTreeFileFormat + * This file is copied from Delta 2.3.0, it is modified to overcome the following issues: + * 1. return ClickhouseOptimisticTransaction + * 2. return DeltaMergeTreeFileFormat + * 3. create HadoopFsRelation with the bucket options */ - /** - * Used to query the current state of the log as well as modify it by adding new atomic collections - * of actions. + * Used to query the current state of the log as well as modify it by adding + * new atomic collections of actions. * - * Internally, this class implements an optimistic concurrency control algorithm to handle multiple - * readers or writers. Any single read is guaranteed to see a consistent snapshot of the table. + * Internally, this class implements an optimistic concurrency control + * algorithm to handle multiple readers or writers. Any single read + * is guaranteed to see a consistent snapshot of the table. + * + * @param logPath Path of the Delta log JSONs. + * @param dataPath Path of the data files. + * @param options Filesystem options filtered from `allOptions`. + * @param allOptions All options provided by the user, for example via `df.write.option()`. This + * includes but not limited to filesystem and table properties. + * @param clock Clock to be used when starting a new transaction. */ -class DeltaLog private ( +class DeltaLog private( val logPath: Path, val dataPath: Path, val options: Map[String, String], + val allOptions: Map[String, String], val clock: Clock -) extends Checkpoints + ) extends Checkpoints with MetadataCleanup with LogStoreProvider with SnapshotManagement with DeltaFileFormat with ReadChecksum { + import org.apache.spark.sql.delta.util.FileNames._ - import DeltaLog._ - implicit private lazy val _clock = clock + private lazy implicit val _clock = clock protected def spark = SparkSession.active @@ -120,8 +129,7 @@ class DeltaLog private ( /** Delta History Manager containing version and commit history. */ lazy val history = new DeltaHistoryManager( - this, - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) + this, spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) /* --------------- * | Configuration | @@ -129,61 +137,25 @@ class DeltaLog private ( /** * The max lineage length of a Snapshot before Delta forces to build a Snapshot from scratch. - * Delta will build a Snapshot on top of the previous one if it doesn't see a checkpoint. However, - * there is a race condition that when two writers are writing at the same time, a writer may fail - * to pick up checkpoints written by another one, and the lineage will grow and finally cause - * StackOverflowError. Hence we have to force to build a Snapshot from scratch when the lineage - * length is too large to avoid hitting StackOverflowError. + * Delta will build a Snapshot on top of the previous one if it doesn't see a checkpoint. + * However, there is a race condition that when two writers are writing at the same time, + * a writer may fail to pick up checkpoints written by another one, and the lineage will grow + * and finally cause StackOverflowError. Hence we have to force to build a Snapshot from scratch + * when the lineage length is too large to avoid hitting StackOverflowError. */ def maxSnapshotLineageLength: Int = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH) - /** How long to keep around logically deleted files before physically deleting them. */ - private[delta] def tombstoneRetentionMillis: Long = - DeltaConfigs.getMilliSeconds(DeltaConfigs.TOMBSTONE_RETENTION.fromMetaData(metadata)) - - // TODO: There is a race here where files could get dropped when increasing the - // retention interval... - protected def metadata = Option(unsafeVolatileSnapshot).map(_.metadata).getOrElse(Metadata()) - - /** - * Tombstones before this timestamp will be dropped from the state and the files can be garbage - * collected. - */ - def minFileRetentionTimestamp: Long = { - // TODO (Fred): Get rid of this FrameProfiler record once SC-94033 is addressed - recordFrameProfile("Delta", "DeltaLog.minFileRetentionTimestamp") { - clock.getTimeMillis() - tombstoneRetentionMillis - } - } - - /** - * [[SetTransaction]]s before this timestamp will be considered expired and dropped from the - * state, but no files will be deleted. - */ - def minSetTransactionRetentionTimestamp: Option[Long] = { - DeltaLog.minSetTransactionRetentionInterval(metadata).map(clock.getTimeMillis() - _) - } - - /** - * Checks whether this table only accepts appends. If so it will throw an error in operations that - * can remove data such as DELETE/UPDATE/MERGE. - */ - def assertRemovable(): Unit = { - if (DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata)) { - throw DeltaErrors.modifyAppendOnlyTableException(metadata.name) - } - } - /** The unique identifier for this table. */ - def tableId: String = metadata.id + def tableId: String = unsafeVolatileMetadata.id // safe because table id never changes /** - * Combines the tableId with the path of the table to ensure uniqueness. Normally `tableId` should - * be globally unique, but nothing stops users from copying a Delta table directly to a separate - * location, where the transaction log is copied directly, causing the tableIds to match. When - * users mutate the copied table, and then try to perform some checks joining the two tables, - * optimizations that depend on `tableId` alone may not be correct. Hence we use a composite id. + * Combines the tableId with the path of the table to ensure uniqueness. Normally `tableId` + * should be globally unique, but nothing stops users from copying a Delta table directly to + * a separate location, where the transaction log is copied directly, causing the tableIds to + * match. When users mutate the copied table, and then try to perform some checks joining the + * two tables, optimizations that depend on `tableId` alone may not be correct. Hence we use a + * composite id. */ private[delta] def compositeId: (String, Path) = tableId -> dataPath @@ -224,19 +196,32 @@ class DeltaLog private ( "ignoreCorruptFiles" -> "false", "ignoreMissingFiles" -> "false" ) - val fsRelation = - HadoopFsRelation(index, index.partitionSchema, schema, None, index.format, allOptions)(spark) + // --- modified start + // Don't need to add the bucketOption here, it handles the delta log meta json file + // --- modified end + val fsRelation = HadoopFsRelation( + index, index.partitionSchema, schema, None, index.format, allOptions)(spark) LogicalRelation(fsRelation) } + /** + * Load the data using the FileIndex. This allows us to skip many checks that add overhead, e.g. + * file existence checks, partitioning schema inference. + */ + def loadIndex( + index: DeltaLogFileIndex, + schema: StructType = Action.logSchema): DataFrame = { + Dataset.ofRows(spark, indexToRelation(index, schema)) + } + /* ------------------ * | Delta Management | * ------------------ */ /** - * Returns a new [[OptimisticTransaction]] that can be used to read the current state of the log - * and then commit updates. The reads and updates will be checked for logical conflicts with any - * concurrent writes to the log. + * Returns a new [[OptimisticTransaction]] that can be used to read the current state of the + * log and then commit updates. The reads and updates will be checked for logical conflicts + * with any concurrent writes to the log. * * Note that all reads in a transaction must go through the returned transaction object, and not * directly to the [[DeltaLog]] otherwise they will not be checked for conflicts. @@ -244,17 +229,18 @@ class DeltaLog private ( def startTransaction(): OptimisticTransaction = startTransaction(None) def startTransaction(snapshotOpt: Option[Snapshot]): OptimisticTransaction = { + // --- modified start new ClickhouseOptimisticTransaction(this, snapshotOpt) + // --- modified end } /** - * Execute a piece of code within a new [[OptimisticTransaction]]. Reads/write sets will be - * recorded for this table, and all other tables will be read at a snapshot that is pinned on the - * first access. + * Execute a piece of code within a new [[OptimisticTransaction]]. Reads/write sets will + * be recorded for this table, and all other tables will be read + * at a snapshot that is pinned on the first access. * - * @note - * This uses thread-local variable to make the active transaction visible. So do not use - * multi-threaded code in the provided thunk. + * @note This uses thread-local variable to make the active transaction visible. So do not use + * multi-threaded code in the provided thunk. */ def withNewTransaction[T](thunk: OptimisticTransaction => T): T = { try { @@ -266,16 +252,16 @@ class DeltaLog private ( } } + /** * Upgrade the table's protocol version, by default to the maximum recognized reader and writer * versions in this DBR release. */ - def upgradeProtocol(snapshot: Snapshot, newVersion: Protocol): Unit = { + def upgradeProtocol( + snapshot: Snapshot, + newVersion: Protocol): Unit = { val currentVersion = snapshot.protocol - if ( - newVersion.minReaderVersion == currentVersion.minReaderVersion && - newVersion.minWriterVersion == currentVersion.minWriterVersion - ) { + if (newVersion == currentVersion) { logConsole(s"Table $dataPath is already at protocol version $newVersion.") return } @@ -292,7 +278,7 @@ class DeltaLog private ( } // Test-only!! - private[delta] def upgradeProtocol(newVersion: Protocol = Protocol()): Unit = { + private[delta] def upgradeProtocol(newVersion: Protocol): Unit = { upgradeProtocol(unsafeVolatileSnapshot, newVersion) } @@ -304,41 +290,39 @@ class DeltaLog private ( startVersion: Long, failOnDataLoss: Boolean = false): Iterator[(Long, Seq[Action])] = { val hadoopConf = newDeltaHadoopConf() - val deltas = store.listFrom(deltaFile(logPath, startVersion), hadoopConf).filter(isDeltaFile) + val deltas = store.listFrom(listingPrefix(logPath, startVersion), hadoopConf) + .filter(isDeltaFile) // Subtract 1 to ensure that we have the same check for the inclusive startVersion var lastSeenVersion = startVersion - 1 - deltas.map { - status => - val p = status.getPath - val version = deltaVersion(p) - if (failOnDataLoss && version > lastSeenVersion + 1) { - throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) - } - lastSeenVersion = version - (version, store.read(status, hadoopConf).map(Action.fromJson)) + deltas.map { status => + val p = status.getPath + val version = deltaVersion(p) + if (failOnDataLoss && version > lastSeenVersion + 1) { + throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) + } + lastSeenVersion = version + (version, store.read(status, hadoopConf).map(Action.fromJson)) } } /** - * Get access to all actions starting from "startVersion" (inclusive) via [[FileStatus]]. If - * `startVersion` doesn't exist, return an empty Iterator. + * Get access to all actions starting from "startVersion" (inclusive) via [[FileStatus]]. + * If `startVersion` doesn't exist, return an empty Iterator. */ def getChangeLogFiles( startVersion: Long, failOnDataLoss: Boolean = false): Iterator[(Long, FileStatus)] = { - val deltas = store - .listFrom(deltaFile(logPath, startVersion), newDeltaHadoopConf()) + val deltas = store.listFrom(listingPrefix(logPath, startVersion), newDeltaHadoopConf()) .filter(isDeltaFile) // Subtract 1 to ensure that we have the same check for the inclusive startVersion var lastSeenVersion = startVersion - 1 - deltas.map { - status => - val version = deltaVersion(status) - if (failOnDataLoss && version > lastSeenVersion + 1) { - throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) - } - lastSeenVersion = version - (version, status) + deltas.map { status => + val version = deltaVersion(status) + if (failOnDataLoss && version > lastSeenVersion + 1) { + throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) + } + lastSeenVersion = version + (version, status) } } @@ -346,40 +330,108 @@ class DeltaLog private ( | Protocol validation | * --------------------- */ + /** + * Asserts the highest protocol supported by this client is not less than what required by the + * table for performing read or write operations. This ensures the client to support a + * greater-or-equal protocol versions and recognizes/supports all features enabled by the table. + * + * The operation type to be checked is passed as a string in `readOrWrite`. Valid values are + * `read` and `write`. + */ + private def protocolCheck(tableProtocol: Protocol, readOrWrite: String): Unit = { + val clientSupportedProtocol = Action.supportedProtocolVersion() + // Depending on the operation, pull related protocol versions out of Protocol objects. + // `getEnabledFeatures` is a pointer to pull reader/writer features out of a Protocol. + val (clientSupportedVersion, tableRequiredVersion, getEnabledFeatures) = readOrWrite match { + case "read" => ( + clientSupportedProtocol.minReaderVersion, + tableProtocol.minReaderVersion, + (f: Protocol) => f.readerFeatureNames) + case "write" => ( + clientSupportedProtocol.minWriterVersion, + tableProtocol.minWriterVersion, + (f: Protocol) => f.writerFeatureNames) + case _ => + throw new IllegalArgumentException("Table operation must be either `read` or `write`.") + } + + // Check is complete when both the protocol version and all referenced features are supported. + val clientSupportedFeatureNames = getEnabledFeatures(clientSupportedProtocol) + val tableEnabledFeatureNames = getEnabledFeatures(tableProtocol) + if (tableEnabledFeatureNames.subsetOf(clientSupportedFeatureNames) && + clientSupportedVersion >= tableRequiredVersion) { + return + } + + // Otherwise, either the protocol version, or few features referenced by the table, is + // unsupported. + val clientUnsupportedFeatureNames = + tableEnabledFeatureNames.diff(clientSupportedFeatureNames) + // Prepare event log constants and the appropriate error message handler. + val (opType, versionKey, unsupportedFeaturesException) = readOrWrite match { + case "read" => ( + "delta.protocol.failure.read", + "minReaderVersion", + DeltaErrors.unsupportedReaderTableFeaturesInTableException _) + case "write" => ( + "delta.protocol.failure.write", + "minWriterVersion", + DeltaErrors.unsupportedWriterTableFeaturesInTableException _) + } + recordDeltaEvent( + this, + opType, + data = Map( + "clientVersion" -> clientSupportedVersion, + versionKey -> tableRequiredVersion, + "clientFeatures" -> clientSupportedFeatureNames.mkString(","), + "clientUnsupportedFeatures" -> clientUnsupportedFeatureNames.mkString(","))) + if (clientSupportedVersion < tableRequiredVersion) { + throw new InvalidProtocolVersionException(tableRequiredVersion, clientSupportedVersion) + } else { + throw unsupportedFeaturesException(clientUnsupportedFeatureNames) + } + } + + /** + * Asserts that the table's protocol enabled all features that are active in the metadata. + * + * A mismatch shouldn't happen when the table has gone through a proper write process because we + * require all active features during writes. However, other clients may void this guarantee. + */ + def assertTableFeaturesMatchMetadata( + targetProtocol: Protocol, + targetMetadata: Metadata): Unit = { + if (!targetProtocol.supportsReaderFeatures && !targetProtocol.supportsWriterFeatures) return + + val protocolEnabledFeatures = targetProtocol.writerFeatureNames + .flatMap(TableFeature.featureNameToFeature) + val activeFeatures: Set[TableFeature] = + TableFeature.allSupportedFeaturesMap.values.collect { + case f: TableFeature with FeatureAutomaticallyEnabledByMetadata + if f.metadataRequiresFeatureToBeEnabled(targetMetadata, spark) => + f + }.toSet + val activeButNotEnabled = activeFeatures.diff(protocolEnabledFeatures) + if (activeButNotEnabled.nonEmpty) { + throw DeltaErrors.tableFeatureMismatchException(activeButNotEnabled.map(_.name)) + } + } + /** * Asserts that the client is up to date with the protocol and allowed to read the table that is * using the given `protocol`. */ def protocolRead(protocol: Protocol): Unit = { - val supportedReaderVersion = - Action.supportedProtocolVersion(Some(spark.sessionState.conf)).minReaderVersion - if (supportedReaderVersion < protocol.minReaderVersion) { - recordDeltaEvent( - this, - "delta.protocol.failure.read", - data = Map( - "clientVersion" -> supportedReaderVersion, - "minReaderVersion" -> protocol.minReaderVersion)) - throw new InvalidProtocolVersionException - } + protocolCheck(protocol, "read") } /** - * Asserts that the client is up to date with the protocol and allowed to write to the table that - * is using the given `protocol`. + * Asserts that the client is up to date with the protocol and allowed to write to the table + * that is using the given `protocol`. */ - def protocolWrite(protocol: Protocol, logUpgradeMessage: Boolean = true): Unit = { - val supportedWriterVersion = - Action.supportedProtocolVersion(Some(spark.sessionState.conf)).minWriterVersion - if (supportedWriterVersion < protocol.minWriterVersion) { - recordDeltaEvent( - this, - "delta.protocol.failure.write", - data = Map( - "clientVersion" -> supportedWriterVersion, - "minWriterVersion" -> protocol.minWriterVersion)) - throw new InvalidProtocolVersionException - } + def protocolWrite(protocol: Protocol): Unit = { + protocolCheck(protocol, "write") } /* ---------------------------------------- * @@ -387,9 +439,10 @@ class DeltaLog private ( * ---------------------------------------- */ /** - * Whether a Delta table exists at this directory. It is okay to use the cached volatile snapshot - * here, since the worst case is that the table has recently started existing which hasn't been - * picked up here. If so, any subsequent command that updates the table will see the right value. + * Whether a Delta table exists at this directory. + * It is okay to use the cached volatile snapshot here, since the worst case is that the table + * has recently started existing which hasn't been picked up here. If so, any subsequent command + * that updates the table will see the right value. */ def tableExists: Boolean = unsafeVolatileSnapshot.version >= 0 @@ -420,38 +473,46 @@ class DeltaLog private ( /** * Returns a [[org.apache.spark.sql.DataFrame]] containing the new files within the specified * version range. + * */ def createDataFrame( snapshot: Snapshot, addFiles: Seq[AddFile], isStreaming: Boolean = false, - actionTypeOpt: Option[String] = None): DataFrame = { + actionTypeOpt: Option[String] = None + ): DataFrame = { val actionType = actionTypeOpt.getOrElse(if (isStreaming) "streaming" else "batch") val fileIndex = new TahoeBatchFileIndex(spark, actionType, addFiles, this, dataPath, snapshot) val hadoopOptions = snapshot.metadata.format.options ++ options + val partitionSchema = snapshot.metadata.partitionSchema + val metadata = snapshot.metadata + val relation = HadoopFsRelation( fileIndex, - partitionSchema = - DeltaColumnMapping.dropColumnMappingMetadata(snapshot.metadata.partitionSchema), + partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata(partitionSchema), // We pass all table columns as `dataSchema` so that Spark will preserve the partition column // locations. Otherwise, for any partition columns not in `dataSchema`, Spark would just // append them to the end of `dataSchema`. dataSchema = DeltaColumnMapping.dropColumnMappingMetadata( - ColumnWithDefaultExprUtils.removeDefaultExpressions(snapshot.metadata.schema)), + ColumnWithDefaultExprUtils.removeDefaultExpressions(metadata.schema)), + // --- modified start + // TODO: Don't add the bucketOption here, it will cause the OOM when the merge into update + // key is the bucket column, fix later + // --- modified end bucketSpec = None, - snapshot.deltaLog.fileFormat(snapshot.metadata), - hadoopOptions - )(spark) + fileFormat(metadata), + hadoopOptions)(spark) Dataset.ofRows(spark, LogicalRelation(relation, isStreaming = isStreaming)) } /** - * Returns a [[BaseRelation]] that contains all of the data present in the table. This relation - * will be continually updated as files are added or removed from the table. However, new - * [[BaseRelation]] must be requested in order to see changes to the schema. + * Returns a [[BaseRelation]] that contains all of the data present + * in the table. This relation will be continually updated + * as files are added or removed from the table. However, new [[BaseRelation]] + * must be requested in order to see changes to the schema. */ def createRelation( partitionFilters: Seq[Expression] = Nil, @@ -473,21 +534,23 @@ class DeltaLog private ( if (!cdcOptions.isEmpty) { recordDeltaEvent(this, "delta.cdf.read", data = cdcOptions.asCaseSensitiveMap()) return CDCReader.getCDCRelation( - spark, - this, - snapshotToUse, - partitionFilters, - spark.sessionState.conf, - cdcOptions) + spark, snapshotToUse, isTimeTravelQuery, spark.sessionState.conf, cdcOptions) } - val fileIndex = - TahoeLogFileIndex(spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) - var bucketSpec: Option[BucketSpec] = ClickHouseTableV2.getTable(this).bucketOption - new DeltaHadoopFsRelation( + val fileIndex = TahoeLogFileIndex( + spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) + // --- modified start + var bucketSpec: Option[BucketSpec] = + if (ClickHouseConfig.isMergeTreeFormatEngine(snapshotToUse.metadata.configuration)) { + ClickHouseTableV2.getTable(this).bucketOption + } else { + None + } + + new DeltaLog.DeltaHadoopFsRelation( fileIndex, - partitionSchema = - DeltaColumnMapping.dropColumnMappingMetadata(snapshotToUse.metadata.partitionSchema), + partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata( + snapshotToUse.metadata.partitionSchema), // We pass all table columns as `dataSchema` so that Spark will preserve the partition column // locations. Otherwise, for any partition columns not in `dataSchema`, Spark would just // append them to the end of `dataSchema` @@ -504,18 +567,21 @@ class DeltaLog private ( spark, this ) + // --- modified end } /** - * Verify the required Spark conf for delta Throw - * `DeltaErrors.configureSparkSessionWithExtensionAndCatalog` exception if - * `spark.sql.catalog.spark_catalog` config is missing. We do not check for `spark.sql.extensions` - * because DeltaSparkSessionExtension can alternatively be activated using the `.withExtension()` - * API. This check can be disabled by setting DELTA_CHECK_REQUIRED_SPARK_CONF to false. + * Verify the required Spark conf for delta + * Throw `DeltaErrors.configureSparkSessionWithExtensionAndCatalog` exception if + * `spark.sql.catalog.spark_catalog` config is missing. We do not check for + * `spark.sql.extensions` because DeltaSparkSessionExtension can alternatively + * be activated using the `.withExtension()` API. This check can be disabled + * by setting DELTA_CHECK_REQUIRED_SPARK_CONF to false. */ protected def checkRequiredConfigurations(): Unit = { if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_REQUIRED_SPARK_CONFS_CHECK)) { - if (spark.conf.getOption(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key).isEmpty) { + if (spark.conf.getOption( + SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key).isEmpty) { throw DeltaErrors.configureSparkSessionWithExtensionAndCatalog(None) } } @@ -524,9 +590,9 @@ class DeltaLog private ( /** * Returns a proper path canonicalization function for the current Delta log. * - * If `runsOnExecutors` is true, the returned method will use a broadcast Hadoop Configuration so - * that the method is suitable for execution on executors. Otherwise, the returned method will use - * a local Hadoop Configuration and the method can only be executed on the driver. + * If `runsOnExecutors` is true, the returned method will use a broadcast Hadoop Configuration + * so that the method is suitable for execution on executors. Otherwise, the returned method + * will use a local Hadoop Configuration and the method can only be executed on the driver. */ private[delta] def getCanonicalPathFunction(runsOnExecutors: Boolean): String => String = { val hadoopConf = newDeltaHadoopConf() @@ -535,7 +601,9 @@ class DeltaLog private ( val broadcastHadoopConf = spark.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) () => broadcastHadoopConf.value.value - } else { () => hadoopConf } + } else { + () => hadoopConf + } new DeltaLog.CanonicalPathFunction(getHadoopConf) } @@ -544,24 +612,33 @@ class DeltaLog private ( * Returns a proper path canonicalization UDF for the current Delta log. * * If `runsOnExecutors` is true, the returned UDF will use a broadcast Hadoop Configuration. - * Otherwise, the returned UDF will use a local Hadoop Configuration and the UDF can only be - * executed on the driver. + * Otherwise, the returned UDF will use a local Hadoop Configuration and the UDF can + * only be executed on the driver. */ private[delta] def getCanonicalPathUdf(runsOnExecutors: Boolean = true): UserDefinedFunction = { DeltaUDF.stringFromString(getCanonicalPathFunction(runsOnExecutors)) } - override def fileFormat(metadata: Metadata = metadata): FileFormat = - ClickHouseTableV2.getTable(this).getFileFormat(metadata) + override def fileFormat(metadata: Metadata): FileFormat = { + // --- modified start + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + ClickHouseTableV2.getTable(this).getFileFormat(metadata) + } else { + super.fileFormat(metadata) + } + // --- modified end + } } object DeltaLog extends DeltaLogging { + + // --- modified start @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) private class DeltaHadoopFsRelation( location: FileIndex, partitionSchema: StructType, - // The top-level columns in `dataSchema` should match the actual physical file schema, otherwise - // the ORC data source may not work with the by-ordinal mode. + // The top-level columns in `dataSchema` should match the actual physical file schema, + // otherwise the ORC data source may not work with the by-ordinal mode. dataSchema: StructType, bucketSpec: Option[BucketSpec], fileFormat: FileFormat, @@ -573,7 +650,7 @@ object DeltaLog extends DeltaLogging { bucketSpec, fileFormat, options)(sparkSession) - with InsertableRelation { + with InsertableRelation { def insert(data: DataFrame, overwrite: Boolean): Unit = { val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append WriteIntoDelta( @@ -586,6 +663,7 @@ object DeltaLog extends DeltaLogging { ).run(sparkSession) } } + // --- modified end /** * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file @@ -602,30 +680,27 @@ object DeltaLog extends DeltaLogging { private[delta] def logPathFor(dataPath: File): Path = logPathFor(dataPath.getAbsolutePath) /** - * We create only a single [[DeltaLog]] for any given `DeltaLogCacheKey` to avoid wasted work in - * reconstructing the log. + * We create only a single [[DeltaLog]] for any given `DeltaLogCacheKey` to avoid wasted work + * in reconstructing the log. */ private val deltaLogCache = { - val builder = CacheBuilder - .newBuilder() + val builder = CacheBuilder.newBuilder() .expireAfterAccess(60, TimeUnit.MINUTES) - .removalListener( - (removalNotification: RemovalNotification[DeltaLogCacheKey, DeltaLog]) => { + .removalListener((removalNotification: RemovalNotification[DeltaLogCacheKey, DeltaLog]) => { val log = removalNotification.getValue // TODO: We should use ref-counting to uncache snapshots instead of a manual timed op - try log.unsafeVolatileSnapshot.uncache() - catch { + try log.unsafeVolatileSnapshot.uncache() catch { case _: java.lang.NullPointerException => // Various layers will throw null pointer if the RDD is already gone. } - }) - sys.props - .get("delta.log.cacheSize") + }) + sys.props.get("delta.log.cacheSize") .flatMap(v => Try(v.toLong).toOption) .foreach(builder.maximumSize) builder.build[DeltaLogCacheKey, DeltaLog]() } + // Don't tolerate malformed JSON when parsing Delta log actions (default is PERMISSIVE) val jsonCommitParseOption = Map("mode" -> FailFastMode.name) @@ -710,28 +785,38 @@ object DeltaLog extends DeltaLogging { private def apply(spark: SparkSession, rawPath: Path, clock: Clock = new SystemClock): DeltaLog = apply(spark, rawPath, Map.empty, clock) + /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot(spark: SparkSession, dataPath: String): (DeltaLog, Snapshot) = - withFreshSnapshot(forTable(spark, dataPath, _)) + withFreshSnapshot { forTable(spark, dataPath, _) } /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot(spark: SparkSession, dataPath: Path): (DeltaLog, Snapshot) = - withFreshSnapshot(forTable(spark, dataPath, _)) + withFreshSnapshot { forTable(spark, dataPath, _) } /** Helper for getting a log, as well as the latest snapshot, of the table */ - def forTableWithSnapshot(spark: SparkSession, tableName: TableIdentifier): (DeltaLog, Snapshot) = - withFreshSnapshot(forTable(spark, tableName, _)) + def forTableWithSnapshot( + spark: SparkSession, + tableName: TableIdentifier): (DeltaLog, Snapshot) = + withFreshSnapshot { forTable(spark, tableName, _) } /** Helper for getting a log, as well as the latest snapshot, of the table */ def forTableWithSnapshot( spark: SparkSession, tableName: DeltaTableIdentifier): (DeltaLog, Snapshot) = - withFreshSnapshot(forTable(spark, tableName, _)) + withFreshSnapshot { forTable(spark, tableName, _) } + + /** Helper for getting a log, as well as the latest snapshot, of the table */ + def forTableWithSnapshot( + spark: SparkSession, + dataPath: Path, + options: Map[String, String]): (DeltaLog, Snapshot) = + withFreshSnapshot { apply(spark, logPathFor(dataPath), options, _) } /** - * Helper function to be used with the forTableWithSnapshot calls. Thunk is a partially applied - * DeltaLog.forTable call, which we can then wrap around with a snapshot update. We use the system - * clock to avoid back-to-back updates. + * Helper function to be used with the forTableWithSnapshot calls. Thunk is a + * partially applied DeltaLog.forTable call, which we can then wrap around with a + * snapshot update. We use the system clock to avoid back-to-back updates. */ private[delta] def withFreshSnapshot(thunk: Clock => DeltaLog): (DeltaLog, Snapshot) = { val clock = new SystemClock @@ -748,14 +833,12 @@ object DeltaLog extends DeltaLogging { clock: Clock ): DeltaLog = { val fileSystemOptions: Map[String, String] = - if ( - spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) - ) { + if (spark.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { // We pick up only file system options so that we don't pass any parquet or json options to // the code that reads Delta transaction logs. - options.filterKeys { - k => DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) + options.filterKeys { k => + DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) }.toMap } else { Map.empty @@ -769,14 +852,15 @@ object DeltaLog extends DeltaLogging { null, "delta.log.create", Map(TAG_TAHOE_PATH -> path.getParent.toString)) { - AnalysisHelper.allowInvokingTransformsInAnalyzer { - new DeltaLog( - logPath = path, - dataPath = path.getParent, - options = fileSystemOptions, - clock = clock - ) - } + AnalysisHelper.allowInvokingTransformsInAnalyzer { + new DeltaLog( + logPath = path, + dataPath = path.getParent, + options = fileSystemOptions, + allOptions = options, + clock = clock + ) + } } def getDeltaLogFromCache(): DeltaLog = { // The following cases will still create a new ActionLog even if there is a cached @@ -785,7 +869,10 @@ object DeltaLog extends DeltaLogging { // - Different `authority` (e.g., different user tokens in the path) // - Different mount point. try { - deltaLogCache.get(path -> fileSystemOptions, () => createDeltaLog()) + deltaLogCache.get(path -> fileSystemOptions, () => { + createDeltaLog() + } + ) } catch { case e: com.google.common.util.concurrent.UncheckedExecutionException => throw e.getCause @@ -814,10 +901,8 @@ object DeltaLog extends DeltaLogging { // scalastyle:on deltahadoopconfiguration val path = fs.makeQualified(rawPath) - if ( - spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) - ) { + if (spark.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { // We rely on the fact that accessing the key set doesn't modify the entry access time. See // `CacheBuilder.expireAfterAccess`. val keysToBeRemoved = mutable.ArrayBuffer[DeltaLogCacheKey]() @@ -848,38 +933,42 @@ object DeltaLog extends DeltaLogging { /** * Filters the given [[Dataset]] by the given `partitionFilters`, returning those that match. - * @param files - * The active files in the DeltaLog state, which contains the partition value information - * @param partitionFilters - * Filters on the partition columns - * @param partitionColumnPrefixes - * The path to the `partitionValues` column, if it's nested + * @param files The active files in the DeltaLog state, which contains the partition value + * information + * @param partitionFilters Filters on the partition columns + * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested + * @param shouldRewritePartitionFilters Whether to rewrite `partitionFilters` to be over the + * [[AddFile]] schema */ def filterFileList( partitionSchema: StructType, files: DataFrame, partitionFilters: Seq[Expression], - partitionColumnPrefixes: Seq[String] = Nil): DataFrame = { - val rewrittenFilters = rewritePartitionFilters( - partitionSchema, - files.sparkSession.sessionState.conf.resolver, - partitionFilters, - partitionColumnPrefixes) + partitionColumnPrefixes: Seq[String] = Nil, + shouldRewritePartitionFilters: Boolean = true): DataFrame = { + + val rewrittenFilters = if (shouldRewritePartitionFilters) { + rewritePartitionFilters( + partitionSchema, + files.sparkSession.sessionState.conf.resolver, + partitionFilters, + partitionColumnPrefixes) + } else { + partitionFilters + } val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) val columnFilter = new Column(expr) files.filter(columnFilter) } /** - * Rewrite the given `partitionFilters` to be used for filtering partition values. We need to - * explicitly resolve the partitioning columns here because the partition columns are stored as - * keys of a Map type instead of attributes in the AddFile schema (below) and thus cannot be - * resolved automatically. + * Rewrite the given `partitionFilters` to be used for filtering partition values. + * We need to explicitly resolve the partitioning columns here because the partition columns + * are stored as keys of a Map type instead of attributes in the AddFile schema (below) and thus + * cannot be resolved automatically. * - * @param partitionFilters - * Filters on the partition columns - * @param partitionColumnPrefixes - * The path to the `partitionValues` column, if it's nested + * @param partitionFilters Filters on the partition columns + * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested */ def rewritePartitionFilters( partitionSchema: StructType, @@ -891,7 +980,7 @@ object DeltaLog extends DeltaLogging { // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. val unquoted = a.name.stripPrefix("`").stripSuffix("`") - val partitionCol = partitionSchema.find(field => resolver(field.name, unquoted)) + val partitionCol = partitionSchema.find { field => resolver(field.name, unquoted) } partitionCol match { case Some(f: StructField) => val name = DeltaColumnMapping.getPhysicalName(f) @@ -907,16 +996,32 @@ object DeltaLog extends DeltaLogging { }) } + + /** + * Checks whether this table only accepts appends. If so it will throw an error in operations that + * can remove data such as DELETE/UPDATE/MERGE. + */ + def assertRemovable(snapshot: Snapshot): Unit = { + val metadata = snapshot.metadata + if (DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata)) { + throw DeltaErrors.modifyAppendOnlyTableException(metadata.name) + } + } + + /** How long to keep around SetTransaction actions before physically deleting them. */ def minSetTransactionRetentionInterval(metadata: Metadata): Option[Long] = { DeltaConfigs.TRANSACTION_ID_RETENTION_DURATION .fromMetaData(metadata) .map(DeltaConfigs.getMilliSeconds) } + /** How long to keep around logically deleted files before physically deleting them. */ + def tombstoneRetentionMillis(metadata: Metadata): Long = { + DeltaConfigs.getMilliSeconds(DeltaConfigs.TOMBSTONE_RETENTION.fromMetaData(metadata)) + } /** Get a function that canonicalizes a given `path`. */ private[delta] class CanonicalPathFunction(getHadoopConf: () => Configuration) - extends Function[String, String] - with Serializable { + extends Function[String, String] with Serializable { // Mark it `@transient lazy val` so that de-serialization happens only once on every executor. @transient private lazy val fs = { diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala similarity index 59% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala index 1c62d133174d..b2b5ba42bb30 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/Snapshot.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala @@ -14,33 +14,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.Expression +// scalastyle:off import.ordering.noEmptyLine +import scala.collection.mutable + import org.apache.spark.sql.delta.actions._ import org.apache.spark.sql.delta.actions.Action.logSchema import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.stats.{DataSkippingReader, DeltaScan, FileSizeHistogram, StatisticsCollection} +import org.apache.spark.sql.delta.stats.DataSkippingReader +import org.apache.spark.sql.delta.stats.DeltaScan +import org.apache.spark.sql.delta.stats.FileSizeHistogram +import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.delta.util.StateCache +import org.apache.hadoop.fs.{FileStatus, Path} + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils -import org.apache.hadoop.fs.{FileStatus, Path} - -// scalastyle:off import.ordering.noEmptyLine -import scala.collection.mutable - /** * Gluten overwrite Delta: * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: - * 1. filesForScan() should return DeltaScan of AddMergeTreeParts instead of AddFile + * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: + * 1. filesForScan() will cache the DeltaScan by the FilterExprsAsKey + * 2. filesForScan() should return DeltaScan of AddMergeTreeParts instead of AddFile */ - /** * A description of a Delta [[Snapshot]], including basic information such its [[DeltaLog]] * metadata, protocol, and version. @@ -55,28 +60,27 @@ trait SnapshotDescriptor { } /** - * An immutable snapshot of the state of the log at some delta version. Internally this class - * manages the replay of actions stored in checkpoint or delta files. + * An immutable snapshot of the state of the log at some delta version. Internally + * this class manages the replay of actions stored in checkpoint or delta files. * - * After resolving any new actions, it caches the result and collects the following basic - * information to the driver: - * - Protocol Version - * - Metadata - * - Transaction state + * After resolving any new actions, it caches the result and collects the + * following basic information to the driver: + * - Protocol Version + * - Metadata + * - Transaction state + * + * @param timestamp The timestamp of the latest commit in milliseconds. Can also be set to -1 if the + * timestamp of the commit is unknown or the table has not been initialized, i.e. + * `version = -1`. * - * @param timestamp - * The timestamp of the latest commit in milliseconds. Can also be set to -1 if the timestamp of - * the commit is unknown or the table has not been initialized, i.e. `version = -1`. */ class Snapshot( val path: Path, override val version: Long, val logSegment: LogSegment, - val minFileRetentionTimestamp: Long, override val deltaLog: DeltaLog, val timestamp: Long, val checksumOpt: Option[VersionChecksum], - val minSetTransactionRetentionTimestamp: Option[Long] = None, checkpointMetadataOpt: Option[CheckpointMetaData] = None) extends SnapshotDescriptor with StateCache @@ -84,25 +88,25 @@ class Snapshot( with DataSkippingReader with DeltaLogging { - import org.apache.spark.sql.delta.implicits._ - - // For implicits which re-use Encoder: import Snapshot._ + // For implicits which re-use Encoder: + import org.apache.spark.sql.delta.implicits._ protected def spark = SparkSession.active + /** Snapshot to scan by the DeltaScanGenerator for metadata query optimizations */ override val snapshotToScan: Snapshot = this protected def getNumPartitions: Int = { - spark.sessionState.conf - .getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS) + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS) .getOrElse(Snapshot.defaultNumSnapshotPartitions) } /** Performs validations during initialization */ protected def init(): Unit = { deltaLog.protocolRead(protocol) + deltaLog.assertTableFeaturesMatchMetadata(protocol, metadata) SchemaUtils.recordUndefinedTypes(deltaLog, metadata.schema) } @@ -127,49 +131,61 @@ class Snapshot( val ADD_PATH_CANONICAL_COL_NAME = "add_path_canonical" val REMOVE_PATH_CANONICAL_COL_NAME = "remove_path_canonical" loadActions - .withColumn( - ADD_PATH_CANONICAL_COL_NAME, - when(col("add.path").isNotNull, canonicalPath(col("add.path")))) - .withColumn( - REMOVE_PATH_CANONICAL_COL_NAME, - when(col("remove.path").isNotNull, canonicalPath(col("remove.path")))) + .withColumn(ADD_PATH_CANONICAL_COL_NAME, when( + col("add.path").isNotNull, canonicalPath(col("add.path")))) + .withColumn(REMOVE_PATH_CANONICAL_COL_NAME, when( + col("remove.path").isNotNull, canonicalPath(col("remove.path")))) .repartition( getNumPartitions, coalesce(col(ADD_PATH_CANONICAL_COL_NAME), col(REMOVE_PATH_CANONICAL_COL_NAME))) .sortWithinPartitions(ACTION_SORT_COL_NAME) - .withColumn( - "add", - when( - col("add.path").isNotNull, - struct( - col(ADD_PATH_CANONICAL_COL_NAME).as("path"), - col("add.partitionValues"), - col("add.size"), - col("add.modificationTime"), - col("add.dataChange"), - col(ADD_STATS_TO_USE_COL_NAME).as("stats"), - col("add.tags") - ) - ) - ) - .withColumn( - "remove", - when( - col("remove.path").isNotNull, - col("remove").withField("path", col(REMOVE_PATH_CANONICAL_COL_NAME)))) + .withColumn("add", when( + col("add.path").isNotNull, + struct( + col(ADD_PATH_CANONICAL_COL_NAME).as("path"), + col("add.partitionValues"), + col("add.size"), + col("add.modificationTime"), + col("add.dataChange"), + col(ADD_STATS_TO_USE_COL_NAME).as("stats"), + col("add.tags"), + col("add.deletionVector") + ))) + .withColumn("remove", when( + col("remove.path").isNotNull, + col("remove").withField("path", col(REMOVE_PATH_CANONICAL_COL_NAME)))) .as[SingleAction] - .mapPartitions { - iter => - val state: LogReplay = - new InMemoryLogReplay( - localMinFileRetentionTimestamp, - localMinSetTransactionRetentionTimestamp) - state.append(0, iter.map(_.unwrap)) - state.checkpoint.map(_.wrap) + .mapPartitions { iter => + val state: LogReplay = + new InMemoryLogReplay( + localMinFileRetentionTimestamp, + localMinSetTransactionRetentionTimestamp) + state.append(0, iter.map(_.unwrap)) + state.checkpoint.map(_.wrap) } } } + /** + * Pulls the protocol and metadata of the table from the files that are used to compute the + * Snapshot directly--without triggering a full state reconstruction. This is important, because + * state reconstruction depends on protocol and metadata for correctness. + */ + protected def protocolAndMetadataReconstruction(): Array[(Protocol, Metadata)] = { + import implicits._ + + val schemaToUse = Action.logSchema(Set("protocol", "metaData")) + fileIndices.map(deltaLog.loadIndex(_, schemaToUse)) + .reduceOption(_.union(_)).getOrElse(emptyDF) + .withColumn(ACTION_SORT_COL_NAME, input_file_name()) + .select("protocol", "metaData", ACTION_SORT_COL_NAME) + .where("protocol.minReaderVersion is not null or metaData.id is not null") + .as[(Protocol, Metadata, String)] + .collect() + .sortBy(_._3) + .map { case (p, m, _) => p -> m } + } + def redactedPath: String = Utils.redact(spark.sessionState.conf.stringRedactionPattern, path.toUri.toString) @@ -189,7 +205,9 @@ class Snapshot( cachedState.getDF } - /** A Map of alias to aggregations which needs to be done to calculate the `computedState` */ + /** + * A Map of alias to aggregations which needs to be done to calculate the `computedState` + */ protected def aggregationsToComputeState: Map[String, Column] = { Map( // sum may return null for empty data set. @@ -223,24 +241,75 @@ class Snapshot( recordDeltaEvent( deltaLog, opType = "delta.assertions.missingAction", - data = - Map("version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) + data = Map( + "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) + throw DeltaErrors.actionNotFoundException("protocol", version) + } else if (_computedState.protocol != protocol) { + recordDeltaEvent( + deltaLog, + opType = "delta.assertions.mismatchedAction", + data = Map( + "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot", + "computedState.protocol" -> _computedState.protocol, + "extracted.protocol" -> protocol)) throw DeltaErrors.actionNotFoundException("protocol", version) } + if (_computedState.metadata == null) { recordDeltaEvent( deltaLog, opType = "delta.assertions.missingAction", - data = - Map("version" -> version.toString, "action" -> "Metadata", "source" -> "Metadata")) + data = Map( + "version" -> version.toString, "action" -> "Metadata", "source" -> "Metadata")) + throw DeltaErrors.actionNotFoundException("metadata", version) + } else if (_computedState.metadata != metadata) { + recordDeltaEvent( + deltaLog, + opType = "delta.assertions.mismatchedAction", + data = Map( + "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot", + "computedState.metadata" -> _computedState.metadata, + "extracted.metadata" -> metadata)) throw DeltaErrors.actionNotFoundException("metadata", version) - } else { - _computedState } + + _computedState } } } + // Used by [[protocol]] and [[metadata]] below + private lazy val (_protocol, _metadata): (Protocol, Metadata) = { + // Should be small. At most 'checkpointInterval' rows, unless new commits are coming + // in before a checkpoint can be written + var protocol: Protocol = null + var metadata: Metadata = null + protocolAndMetadataReconstruction().foreach { + case (p: Protocol, _) => protocol = p + case (_, m: Metadata) => metadata = m + } + + if (protocol == null) { + recordDeltaEvent( + deltaLog, + opType = "delta.assertions.missingAction", + data = Map( + "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) + throw DeltaErrors.actionNotFoundException("protocol", version) + } + + if (metadata == null) { + recordDeltaEvent( + deltaLog, + opType = "delta.assertions.missingAction", + data = Map( + "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot")) + throw DeltaErrors.actionNotFoundException("metadata", version) + } + + protocol -> metadata + } + def sizeInBytes: Long = computedState.sizeInBytes def numOfSetTransactions: Long = computedState.numOfSetTransactions def numOfFiles: Long = computedState.numOfFiles @@ -248,18 +317,34 @@ class Snapshot( def numOfMetadata: Long = computedState.numOfMetadata def numOfProtocol: Long = computedState.numOfProtocol def setTransactions: Seq[SetTransaction] = computedState.setTransactions - override def metadata: Metadata = computedState.metadata - override def protocol: Protocol = computedState.protocol + override def metadata: Metadata = _metadata + override def protocol: Protocol = _protocol def fileSizeHistogram: Option[FileSizeHistogram] = computedState.fileSizeHistogram - private[delta] def sizeInBytesOpt: Option[Long] = Some(sizeInBytes) - private[delta] def setTransactionsOpt: Option[Seq[SetTransaction]] = Some(setTransactions) - private[delta] def numOfFilesOpt: Option[Long] = Some(numOfFiles) + private[delta] def sizeInBytesIfKnown: Option[Long] = Some(sizeInBytes) + private[delta] def setTransactionsIfKnown: Option[Seq[SetTransaction]] = Some(setTransactions) + private[delta] def numOfFilesIfKnown: Option[Long] = Some(numOfFiles) + + /** + * Tombstones before the [[minFileRetentionTimestamp]] timestamp will be dropped from the + * checkpoint. + */ + private[delta] def minFileRetentionTimestamp: Long = { + deltaLog.clock.getTimeMillis() - DeltaLog.tombstoneRetentionMillis(metadata) + } + + /** + * [[SetTransaction]]s before [[minSetTransactionRetentionTimestamp]] will be considered expired + * and dropped from the snapshot. + */ + private[delta] def minSetTransactionRetentionTimestamp: Option[Long] = { + DeltaLog.minSetTransactionRetentionInterval(metadata).map(deltaLog.clock.getTimeMillis() - _) + } /** - * Computes all the information that is needed by the checksum for the current snapshot. May kick - * off state reconstruction if needed by any of the underlying fields. Note that it's safe to set - * txnId to none, since the snapshot doesn't always have a txn attached. E.g. if a snapshot is - * created by reading a checkpoint, then no txnId is present. + * Computes all the information that is needed by the checksum for the current snapshot. + * May kick off state reconstruction if needed by any of the underlying fields. + * Note that it's safe to set txnId to none, since the snapshot doesn't always have a txn + * attached. E.g. if a snapshot is created by reading a checkpoint, then no txnId is present. */ def computeChecksum: VersionChecksum = VersionChecksum( txnId = None, @@ -271,8 +356,7 @@ class Snapshot( metadata = metadata, protocol = protocol, histogramOpt = fileSizeHistogram, - allFiles = checksumOpt.flatMap(_.allFiles) - ) + allFiles = checksumOpt.flatMap(_.allFiles)) /** A map to look up transaction version by appId. */ lazy val transactions: Map[String, Long] = setTransactions.map(t => t.appId -> t.version).toMap @@ -300,17 +384,23 @@ class Snapshot( lazy val numIndexedCols: Int = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(metadata) /** Return the set of properties of the table. */ - def getProperties: mutable.HashMap[String, String] = { - val base = new mutable.HashMap[String, String]() - metadata.configuration.foreach { - case (k, v) => - if (k != "path") { - base.put(k, v) - } + def getProperties: mutable.Map[String, String] = { + val base = new mutable.LinkedHashMap[String, String]() + metadata.configuration.foreach { case (k, v) => + if (k != "path") { + base.put(k, v) + } } base.put(Protocol.MIN_READER_VERSION_PROP, protocol.minReaderVersion.toString) base.put(Protocol.MIN_WRITER_VERSION_PROP, protocol.minWriterVersion.toString) - base + if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { + val features = protocol.readerAndWriterFeatureNames.map(name => + s"${TableFeatureProtocolUtils.FEATURE_PROP_PREFIX}$name" -> + TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED) + base ++ features.toSeq.sorted + } else { + base + } } // Given the list of files from `LogSegment`, create respective file indices to help create @@ -345,10 +435,8 @@ class Snapshot( * config settings for delta.checkpoint.writeStatsAsJson and delta.checkpoint.writeStatsAsStruct). */ protected def loadActions: DataFrame = { - val dfs = fileIndices.map(index => Dataset.ofRows(spark, deltaLog.indexToRelation(index))) - dfs - .reduceOption(_.union(_)) - .getOrElse(emptyDF) + fileIndices.map(deltaLog.loadIndex(_)) + .reduceOption(_.union(_)).getOrElse(emptyDF) .withColumn(ACTION_SORT_COL_NAME, input_file_name()) .withColumn(ADD_STATS_TO_USE_COL_NAME, col("add.stats")) } @@ -356,6 +444,7 @@ class Snapshot( protected def emptyDF: DataFrame = spark.createDataFrame(spark.sparkContext.emptyRDD[Row], logSchema) + override def logInfo(msg: => String): Unit = { super.logInfo(s"[tableId=${deltaLog.tableId}] " + msg) } @@ -380,21 +469,22 @@ class Snapshot( s"${getClass.getSimpleName}(path=$path, version=$version, metadata=$metadata, " + s"logSegment=$logSegment, checksumOpt=$checksumOpt)" - override def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean): DeltaScan = { + // --- modified start + override def filesForScan(limit: Long): DeltaScan = { val deltaScan = ClickhouseSnapshot.deltaScanCache.get( - FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), filters, None), + FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), Seq.empty, Some(limit)), () => { - super.filesForScan(filters, keepNumRecords) + super.filesForScan(limit) }) replaceWithAddMergeTreeParts(deltaScan) } - override def filesForScan(limit: Long): DeltaScan = { + override def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean): DeltaScan = { val deltaScan = ClickhouseSnapshot.deltaScanCache.get( - FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), Seq.empty, Some(limit)), + FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), filters, None), () => { - super.filesForScan(limit) + super.filesForScan(filters, keepNumRecords) }) replaceWithAddMergeTreeParts(deltaScan) @@ -411,30 +501,35 @@ class Snapshot( } private def replaceWithAddMergeTreeParts(deltaScan: DeltaScan) = { - DeltaScan.apply( - deltaScan.version, - deltaScan.files - .map( - addFile => { - val addFileAsKey = AddFileAsKey(addFile) - - val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) - // this is for later use - ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) - ret - }), - deltaScan.total, - deltaScan.partition, - deltaScan.scanned - )( - deltaScan.scannedSnapshot, - deltaScan.partitionFilters, - deltaScan.dataFilters, - deltaScan.unusedFilters, - deltaScan.scanDurationMs, - deltaScan.dataSkippingType - ) + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + DeltaScan.apply( + deltaScan.version, + deltaScan.files + .map( + addFile => { + val addFileAsKey = AddFileAsKey(addFile) + + val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) + // this is for later use + ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) + ret + }), + deltaScan.total, + deltaScan.partition, + deltaScan.scanned + )( + deltaScan.scannedSnapshot, + deltaScan.partitionFilters, + deltaScan.dataFilters, + deltaScan.unusedFilters, + deltaScan.scanDurationMs, + deltaScan.dataSkippingType + ) + } else { + deltaScan + } } + // --- modified end logInfo(s"Created snapshot $this") init() @@ -450,64 +545,51 @@ object Snapshot extends DeltaLogging { /** Verifies that a set of delta or checkpoint files to be read actually belongs to this table. */ private def assertLogFilesBelongToTable(logBasePath: Path, files: Seq[FileStatus]): Unit = { - files.map(_.getPath).foreach { - filePath => - if (new Path(filePath.toUri).getParent != new Path(logBasePath.toUri)) { - // scalastyle:off throwerror - throw new AssertionError( - s"File ($filePath) doesn't belong in the " + - s"transaction log at $logBasePath. Please contact Databricks Support.") - // scalastyle:on throwerror - } + files.map(_.getPath).foreach { filePath => + if (new Path(filePath.toUri).getParent != new Path(logBasePath.toUri)) { + // scalastyle:off throwerror + throw new AssertionError(s"File ($filePath) doesn't belong in the " + + s"transaction log at $logBasePath. Please contact Databricks Support.") + // scalastyle:on throwerror + } } } /** * Metrics and metadata computed around the Delta table. - * @param sizeInBytes - * The total size of the table (of active files, not including tombstones). - * @param numOfSetTransactions - * Number of streams writing to this table. - * @param numOfFiles - * The number of files in this table. - * @param numOfRemoves - * The number of tombstones in the state. - * @param numOfMetadata - * The number of metadata actions in the state. Should be 1. - * @param numOfProtocol - * The number of protocol actions in the state. Should be 1. - * @param setTransactions - * The streaming queries writing to this table. - * @param metadata - * The metadata of the table. - * @param protocol - * The protocol version of the Delta table. - * @param fileSizeHistogram - * A Histogram class tracking the file counts and total bytes in different size ranges. + * @param sizeInBytes The total size of the table (of active files, not including tombstones). + * @param numOfSetTransactions Number of streams writing to this table. + * @param numOfFiles The number of files in this table. + * @param numOfRemoves The number of tombstones in the state. + * @param numOfMetadata The number of metadata actions in the state. Should be 1. + * @param numOfProtocol The number of protocol actions in the state. Should be 1. + * @param setTransactions The streaming queries writing to this table. + * @param metadata The metadata of the table. + * @param protocol The protocol version of the Delta table. + * @param fileSizeHistogram A Histogram class tracking the file counts and total bytes + * in different size ranges. */ case class State( - sizeInBytes: Long, - numOfSetTransactions: Long, - numOfFiles: Long, - numOfRemoves: Long, - numOfMetadata: Long, - numOfProtocol: Long, - setTransactions: Seq[SetTransaction], - metadata: Metadata, - protocol: Protocol, - fileSizeHistogram: Option[FileSizeHistogram] = None) + sizeInBytes: Long, + numOfSetTransactions: Long, + numOfFiles: Long, + numOfRemoves: Long, + numOfMetadata: Long, + numOfProtocol: Long, + setTransactions: Seq[SetTransaction], + metadata: Metadata, + protocol: Protocol, + fileSizeHistogram: Option[FileSizeHistogram] = None + ) } /** * An initial snapshot with only metadata specified. Useful for creating a DataFrame from an * existing parquet table during its conversion to delta. * - * @param logPath - * the path to transaction log - * @param deltaLog - * the delta log object - * @param metadata - * the metadata of the table + * @param logPath the path to transaction log + * @param deltaLog the delta log object + * @param metadata the metadata of the table */ class InitialSnapshot( val logPath: Path, @@ -517,27 +599,30 @@ class InitialSnapshot( path = logPath, version = -1, logSegment = LogSegment.empty(logPath), - minFileRetentionTimestamp = -1, deltaLog = deltaLog, timestamp = -1, - checksumOpt = None, - minSetTransactionRetentionTimestamp = None + checksumOpt = None ) { def this(logPath: Path, deltaLog: DeltaLog) = this( logPath, deltaLog, Metadata( - configuration = - DeltaConfigs.mergeGlobalConfigs(SparkSession.active.sessionState.conf, Map.empty), - createdTime = Some(System.currentTimeMillis())) - ) + configuration = DeltaConfigs.mergeGlobalConfigs( + sqlConfs = SparkSession.active.sessionState.conf, + tableConf = Map.empty, + ignoreProtocolConfsOpt = Some( + DeltaConfigs.ignoreProtocolDefaultsIsSet( + sqlConfs = SparkSession.active.sessionState.conf, + tableConf = deltaLog.allOptions))), + createdTime = Some(System.currentTimeMillis()))) override def stateDS: Dataset[SingleAction] = emptyDF.as[SingleAction] override def stateDF: DataFrame = emptyDF override protected lazy val computedState: Snapshot.State = initialState + override def protocol: Protocol = computedState.protocol private def initialState: Snapshot.State = { - val protocol = Protocol.forNewTable(spark, metadata) + val protocol = Protocol.forNewTable(spark, Some(metadata)) Snapshot.State( sizeInBytes = 0L, numOfSetTransactions = 0L, @@ -550,5 +635,4 @@ class InitialSnapshot( protocol = protocol ) } - } diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 61% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala index 006a3fce8429..5f9c2953ba16 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/DeleteCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala @@ -14,33 +14,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta.commands -import org.apache.spark.SparkContext -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, EqualNullSafe, Expression, If, Literal, Not} -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction} import org.apache.spark.sql.delta.commands.DeleteCommand.{rewritingFilesMsg, FINDING_TOUCHED_FILES_MSG} import org.apache.spark.sql.delta.commands.MergeIntoCommand.totalBytesAndDistinctPartitionValues import org.apache.spark.sql.delta.files.TahoeBatchFileIndex import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.execution.SQLExecution +import org.apache.spark.sql.delta.util.Utils +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + +import org.apache.spark.SparkContext +import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, EqualNullSafe, Expression, If, Literal, Not} +import org.apache.spark.sql.catalyst.plans.QueryPlan +import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} import org.apache.spark.sql.functions.{col, explode, input_file_name, split} import org.apache.spark.sql.types.LongType -import com.fasterxml.jackson.databind.annotation.JsonDeserialize - /** * Gluten overwrite Delta: * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: + * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: * 1. In Clickhouse backend, we can't implement input_file_name() correctly, we can only implement * it so that it return a a list of filenames (concated by ','). */ @@ -60,8 +61,8 @@ trait DeleteCommandMetrics { self: LeafRunnableCommand => "numPartitionsAddedTo" -> createMetric(sc, "number of partitions added"), "numPartitionsRemovedFrom" -> createMetric(sc, "number of partitions removed"), "numCopiedRows" -> createMetric(sc, "number of rows copied"), - "numBytesAdded" -> createMetric(sc, "number of bytes added"), - "numBytesRemoved" -> createMetric(sc, "number of bytes removed"), + "numAddedBytes" -> createMetric(sc, "number of bytes added"), + "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), "executionTimeMs" -> createTimingMetric(sc, "time taken to execute the entire operation"), "scanTimeMs" -> @@ -73,7 +74,7 @@ trait DeleteCommandMetrics { self: LeafRunnableCommand => "numTouchedRows" -> createMetric(sc, "number of rows touched") ) - def getDeletedRowsFromAddFilesAndUpdateMetrics(files: Seq[AddFile]): Option[Long] = { + def getDeletedRowsFromAddFilesAndUpdateMetrics(files: Seq[AddFile]) : Option[Long] = { if (!conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA)) { return None; } @@ -97,15 +98,18 @@ trait DeleteCommandMetrics { self: LeafRunnableCommand => /** * Performs a Delete based on the search condition * - * Algorithm: 1) Scan all the files and determine which files have the rows that need to be deleted. - * 2) Traverse the affected files and rebuild the touched files. 3) Use the Delta protocol to - * atomically write the remaining rows to new files and remove the affected files that are - * identified in step 1. + * Algorithm: + * 1) Scan all the files and determine which files have + * the rows that need to be deleted. + * 2) Traverse the affected files and rebuild the touched files. + * 3) Use the Delta protocol to atomically write the remaining rows to new files and remove + * the affected files that are identified in step 1. */ -case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Option[Expression]) - extends LeafRunnableCommand - with DeltaCommand - with DeleteCommandMetrics { +case class DeleteCommand( + deltaLog: DeltaLog, + target: LogicalPlan, + condition: Option[Expression]) + extends LeafRunnableCommand with DeltaCommand with DeleteCommandMetrics { override def innerChildren: Seq[QueryPlan[_]] = Seq(target) @@ -115,13 +119,15 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt final override def run(sparkSession: SparkSession): Seq[Row] = { recordDeltaOperation(deltaLog, "delta.dml.delete") { - deltaLog.assertRemovable() - deltaLog.withNewTransaction { - txn => - val deleteActions = performDelete(sparkSession, deltaLog, txn) - if (deleteActions.nonEmpty) { - txn.commit(deleteActions, DeltaOperations.Delete(condition.map(_.sql).toSeq)) - } + deltaLog.withNewTransaction { txn => + DeltaLog.assertRemovable(txn.snapshot) + if (hasBeenExecuted(txn, sparkSession)) { + sendDriverMetrics(sparkSession, metrics) + return Seq.empty + } + + val deleteActions = performDelete(sparkSession, deltaLog, txn) + txn.commitIfNeeded(deleteActions, DeltaOperations.Delete(condition.map(_.sql).toSeq)) } // Re-cache all cached plans(including this relation itself, if it's cached) that refer to // this data source relation. @@ -150,9 +156,9 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt var numAddedChangeFiles: Long = 0 var scanTimeMs: Long = 0 var rewriteTimeMs: Long = 0 - var numBytesAdded: Long = 0 + var numAddedBytes: Long = 0 var changeFileBytes: Long = 0 - var numBytesRemoved: Long = 0 + var numRemovedBytes: Long = 0 var numFilesBeforeSkipping: Long = 0 var numBytesBeforeSkipping: Long = 0 var numFilesAfterSkipping: Long = 0 @@ -175,7 +181,7 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt numRemovedFiles = allFiles.size scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 val (numBytes, numPartitions) = totalBytesAndDistinctPartitionValues(allFiles) - numBytesRemoved = numBytes + numRemovedBytes = numBytes numFilesBeforeSkipping = numRemovedFiles numBytesBeforeSkipping = numBytes numFilesAfterSkipping = numRemovedFiles @@ -192,9 +198,7 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt case Some(cond) => val (metadataPredicates, otherPredicates) = DeltaTableUtils.splitMetadataAndDataPredicates( - cond, - txn.metadata.partitionColumns, - sparkSession) + cond, txn.metadata.partitionColumns, sparkSession) numFilesBeforeSkipping = txn.snapshot.numOfFiles numBytesBeforeSkipping = txn.snapshot.sizeInBytes @@ -209,7 +213,7 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 numRemovedFiles = candidateFiles.size - numBytesRemoved = candidateFiles.map(_.size).sum + numRemovedBytes = candidateFiles.map(_.size).sum numFilesAfterSkipping = candidateFiles.size val (numCandidateBytes, numCandidatePartitions) = totalBytesAndDistinctPartitionValues(candidateFiles) @@ -224,7 +228,15 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt candidateFiles.map(_.removeWithTimestamp(operationTimestamp)) } else { // Case 3: Delete the rows based on the condition. - val candidateFiles = txn.filterFiles(metadataPredicates ++ otherPredicates) + + // Should we write the DVs to represent the deleted rows? + val shouldWriteDVs = shouldWritePersistentDeletionVectors(sparkSession, txn) + + val candidateFiles = txn.filterFiles( + metadataPredicates ++ otherPredicates, + keepNumRecords = shouldWriteDVs) + // `candidateFiles` contains the files filtered using statistics and delete condition + // They may or may not contains any rows that need to be deleted. numFilesAfterSkipping = candidateFiles.size val (numCandidateBytes, numCandidatePartitions) = @@ -237,89 +249,104 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt val nameToAddFileMap = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) val fileIndex = new TahoeBatchFileIndex( - sparkSession, - "delete", - candidateFiles, - deltaLog, - deltaLog.dataPath, - txn.snapshot) - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) - val data = Dataset.ofRows(sparkSession, newTarget) - val deletedRowCount = metrics("numDeletedRows") - val deletedRowUdf = DeltaUDF - .boolean { - () => - deletedRowCount += 1 - true - } - .asNondeterministic() - val filesToRewrite = - withStatusCode("DELTA", FINDING_TOUCHED_FILES_MSG) { - if (candidateFiles.isEmpty) { - Array.empty[String] - } else { - data - .filter(new Column(cond)) - .select(input_file_name().as("input_files")) - .filter(deletedRowUdf()) - .select(explode(split(col("input_files"), ","))) - .distinct() - .as[String] - .collect() - } - } + sparkSession, "delete", candidateFiles, deltaLog, deltaLog.dataPath, txn.snapshot) + if (shouldWriteDVs) { + val targetDf = DeleteWithDeletionVectorsHelper.createTargetDfForScanningForMatches( + sparkSession, + target, + fileIndex) - numRemovedFiles = filesToRewrite.length - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - if (filesToRewrite.isEmpty) { - // Case 3.1: no row matches and no delete will be triggered - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsRemovedFrom = Some(0) - numPartitionsAddedTo = Some(0) - } - Nil - } else { - // Case 3.2: some files need an update to remove the deleted files - // Do the second pass and just read the affected files - val baseRelation = buildBaseRelation( + // Does the target table already has DVs enabled? If so, we need to read the table + // with deletion vectors. + val mustReadDeletionVectors = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) + + val touchedFiles = DeleteWithDeletionVectorsHelper.findTouchedFiles( sparkSession, txn, - "delete", - deltaLog.dataPath, - filesToRewrite, - nameToAddFileMap) + mustReadDeletionVectors, + deltaLog, + targetDf, + fileIndex, + cond) + + if (touchedFiles.nonEmpty) { + DeleteWithDeletionVectorsHelper.processUnmodifiedData(touchedFiles) + } else { + Nil // Nothing to update + } + } else { // Keep everything from the resolved target except a new TahoeFileIndex // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) - val targetDF = Dataset.ofRows(sparkSession, newTarget) - val filterCond = Not(EqualNullSafe(cond, Literal.TrueLiteral)) - val rewrittenActions = rewriteFiles(txn, targetDF, filterCond, filesToRewrite.length) - val (changeFiles, rewrittenFiles) = rewrittenActions - .partition(_.isInstanceOf[AddCDCFile]) - numAddedFiles = rewrittenFiles.size - val removedFiles = - filesToRewrite.map(f => getTouchedFile(deltaLog.dataPath, f, nameToAddFileMap)) - val (removedBytes, removedPartitions) = - totalBytesAndDistinctPartitionValues(removedFiles) - numBytesRemoved = removedBytes - val (rewrittenBytes, rewrittenPartitions) = - totalBytesAndDistinctPartitionValues(rewrittenFiles) - numBytesAdded = rewrittenBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsRemovedFrom = Some(removedPartitions) - numPartitionsAddedTo = Some(rewrittenPartitions) + val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) + val data = Dataset.ofRows(sparkSession, newTarget) + val deletedRowCount = metrics("numDeletedRows") + val deletedRowUdf = DeltaUDF.boolean { () => + deletedRowCount += 1 + true + }.asNondeterministic() + val filesToRewrite = + withStatusCode("DELTA", FINDING_TOUCHED_FILES_MSG) { + if (candidateFiles.isEmpty) { + Array.empty[String] + } else { + // --- modified start + data.filter(new Column(cond)) + .select(input_file_name().as("input_files")) + .filter(deletedRowUdf()) + .select(explode(split(col("input_files"), ","))) + .distinct() + .as[String] + .collect() + // --- modified end + } + } + + numRemovedFiles = filesToRewrite.length + scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 + if (filesToRewrite.isEmpty) { + // Case 3.1: no row matches and no delete will be triggered + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsRemovedFrom = Some(0) + numPartitionsAddedTo = Some(0) + } + Nil + } else { + // Case 3.2: some files need an update to remove the deleted files + // Do the second pass and just read the affected files + val baseRelation = buildBaseRelation( + sparkSession, txn, "delete", deltaLog.dataPath, filesToRewrite, nameToAddFileMap) + // Keep everything from the resolved target except a new TahoeFileIndex + // that only involves the affected files instead of all files. + val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) + val targetDF = Dataset.ofRows(sparkSession, newTarget) + val filterCond = Not(EqualNullSafe(cond, Literal.TrueLiteral)) + val rewrittenActions = rewriteFiles(txn, targetDF, filterCond, filesToRewrite.length) + val (changeFiles, rewrittenFiles) = rewrittenActions + .partition(_.isInstanceOf[AddCDCFile]) + numAddedFiles = rewrittenFiles.size + val removedFiles = filesToRewrite.map(f => + getTouchedFile(deltaLog.dataPath, f, nameToAddFileMap)) + val (removedBytes, removedPartitions) = + totalBytesAndDistinctPartitionValues(removedFiles) + numRemovedBytes = removedBytes + val (rewrittenBytes, rewrittenPartitions) = + totalBytesAndDistinctPartitionValues(rewrittenFiles) + numAddedBytes = rewrittenBytes + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsRemovedFrom = Some(removedPartitions) + numPartitionsAddedTo = Some(rewrittenPartitions) + } + numAddedChangeFiles = changeFiles.size + changeFileBytes = changeFiles.collect { case f: AddCDCFile => f.size }.sum + rewriteTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - scanTimeMs + numDeletedRows = Some(metrics("numDeletedRows").value) + numCopiedRows = + Some(metrics("numTouchedRows").value - metrics("numDeletedRows").value) + + val operationTimestamp = System.currentTimeMillis() + removeFilesFromPaths( + deltaLog, nameToAddFileMap, filesToRewrite, operationTimestamp) ++ rewrittenActions } - numAddedChangeFiles = changeFiles.size - changeFileBytes = changeFiles.collect { case f: AddCDCFile => f.size }.sum - rewriteTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - scanTimeMs - numDeletedRows = Some(metrics("numDeletedRows").value) - numCopiedRows = Some(metrics("numTouchedRows").value - metrics("numDeletedRows").value) - - val operationTimestamp = System.currentTimeMillis() - removeFilesFromPaths(deltaLog, nameToAddFileMap, filesToRewrite, operationTimestamp) ++ - rewrittenActions } } } @@ -331,8 +358,8 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt metrics("rewriteTimeMs").set(rewriteTimeMs) metrics("numAddedChangeFiles").set(numAddedChangeFiles) metrics("changeFileBytes").set(changeFileBytes) - metrics("numBytesAdded").set(numBytesAdded) - metrics("numBytesRemoved").set(numBytesRemoved) + metrics("numAddedBytes").set(numAddedBytes) + metrics("numRemovedBytes").set(numRemovedBytes) metrics("numFilesBeforeSkipping").set(numFilesBeforeSkipping) metrics("numBytesBeforeSkipping").set(numBytesBeforeSkipping) metrics("numFilesAfterSkipping").set(numFilesAfterSkipping) @@ -342,9 +369,7 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt numPartitionsRemovedFrom.foreach(metrics("numPartitionsRemovedFrom").set) numCopiedRows.foreach(metrics("numCopiedRows").set) txn.registerSQLMetrics(sparkSession, metrics) - // This is needed to make the SQL metrics visible in the Spark UI - val executionId = sparkSession.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates(sparkSession.sparkContext, executionId, metrics.values.toSeq) + sendDriverMetrics(sparkSession, metrics) recordDeltaEvent( deltaLog, @@ -366,18 +391,23 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt numPartitionsRemovedFrom, numCopiedRows, numDeletedRows, - numBytesAdded, - numBytesRemoved, + numAddedBytes, + numRemovedBytes, changeFileBytes = changeFileBytes, scanTimeMs, - rewriteTimeMs - ) + rewriteTimeMs) ) - deleteActions + if (deleteActions.nonEmpty) { + createSetTransaction(sparkSession, deltaLog).toSeq ++ deleteActions + } else { + Seq.empty + } } - /** Returns the list of [[AddFile]]s and [[AddCDCFile]]s that have been re-written. */ + /** + * Returns the list of [[AddFile]]s and [[AddCDCFile]]s that have been re-written. + */ private def rewriteFiles( txn: OptimisticTransaction, baseData: DataFrame, @@ -387,15 +417,13 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt // number of total rows that we have seen / are either copying or deleting (sum of both). val numTouchedRows = metrics("numTouchedRows") - val numTouchedRowsUdf = DeltaUDF - .boolean { - () => - numTouchedRows += 1 - true - } - .asNondeterministic() + val numTouchedRowsUdf = DeltaUDF.boolean { () => + numTouchedRows += 1 + true + }.asNondeterministic() - withStatusCode("DELTA", rewritingFilesMsg(numFilesToRewrite)) { + withStatusCode( + "DELTA", rewritingFilesMsg(numFilesToRewrite)) { val dfToWrite = if (shouldWriteCdc) { import org.apache.spark.sql.delta.commands.cdc.CDCReader._ // The logic here ends up being surprisingly elegant, with all source rows ending up in @@ -418,6 +446,14 @@ case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Opt txn.writeFiles(dfToWrite) } } + + def shouldWritePersistentDeletionVectors( + spark: SparkSession, txn: OptimisticTransaction): Boolean = { + // DELETE with DVs only enabled for tests. + Utils.isTesting && + spark.conf.get(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS) && + DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) + } } object DeleteCommand { @@ -441,51 +477,29 @@ object DeleteCommand { /** * Used to report details about delete. * - * @param condition: - * what was the delete condition - * @param numFilesTotal: - * how big is the table - * @param numTouchedFiles: - * how many files did we touch. Alias for `numFilesAfterSkipping` - * @param numRewrittenFiles: - * how many files had to be rewritten. Alias for `numAddedFiles` - * @param numRemovedFiles: - * how many files we removed. Alias for `numTouchedFiles` - * @param numAddedFiles: - * how many files we added. Alias for `numRewrittenFiles` - * @param numAddedChangeFiles: - * how many change files were generated - * @param numFilesBeforeSkipping: - * how many candidate files before skipping - * @param numBytesBeforeSkipping: - * how many candidate bytes before skipping - * @param numFilesAfterSkipping: - * how many candidate files after skipping - * @param numBytesAfterSkipping: - * how many candidate bytes after skipping - * @param numPartitionsAfterSkipping: - * how many candidate partitions after skipping - * @param numPartitionsAddedTo: - * how many new partitions were added - * @param numPartitionsRemovedFrom: - * how many partitions were removed - * @param numCopiedRows: - * how many rows were copied - * @param numDeletedRows: - * how many rows were deleted - * @param numBytesAdded: - * how many bytes were added - * @param numBytesRemoved: - * how many bytes were removed - * @param changeFileBytes: - * total size of change files generated - * @param scanTimeMs: - * how long did finding take - * @param rewriteTimeMs: - * how long did rewriting take + * @param condition: what was the delete condition + * @param numFilesTotal: how big is the table + * @param numTouchedFiles: how many files did we touch. Alias for `numFilesAfterSkipping` + * @param numRewrittenFiles: how many files had to be rewritten. Alias for `numAddedFiles` + * @param numRemovedFiles: how many files we removed. Alias for `numTouchedFiles` + * @param numAddedFiles: how many files we added. Alias for `numRewrittenFiles` + * @param numAddedChangeFiles: how many change files were generated + * @param numFilesBeforeSkipping: how many candidate files before skipping + * @param numBytesBeforeSkipping: how many candidate bytes before skipping + * @param numFilesAfterSkipping: how many candidate files after skipping + * @param numBytesAfterSkipping: how many candidate bytes after skipping + * @param numPartitionsAfterSkipping: how many candidate partitions after skipping + * @param numPartitionsAddedTo: how many new partitions were added + * @param numPartitionsRemovedFrom: how many partitions were removed + * @param numCopiedRows: how many rows were copied + * @param numDeletedRows: how many rows were deleted + * @param numBytesAdded: how many bytes were added + * @param numBytesRemoved: how many bytes were removed + * @param changeFileBytes: total size of change files generated + * @param scanTimeMs: how long did finding take + * @param rewriteTimeMs: how long did rewriting take * - * @note - * All the time units are milliseconds. + * @note All the time units are milliseconds. */ case class DeleteMetric( condition: String, diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala similarity index 70% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala index 5967d66b13b5..bb4d66897565 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala @@ -14,63 +14,66 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta.commands +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} +import org.apache.spark.sql.delta.commands.merge.MergeIntoMaterializeSource +import org.apache.spark.sql.delta.files._ +import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaUtils} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.{AnalysisHelper, SetAccumulator} +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + import org.apache.spark.SparkContext import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, BasePredicate, Expression, Literal, NamedExpression, PredicateHelper, UnsafeProjection} +import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, BasePredicate, Expression, Literal, NamedExpression, PredicateHelper, UnsafeProjection} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.merge.MergeIntoMaterializeSource -import org.apache.spark.sql.delta.files._ -import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaUtils} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.{AnalysisHelper, SetAccumulator} -import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.{DataTypes, LongType, StructType} -import com.fasterxml.jackson.databind.annotation.JsonDeserialize - -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ -import scala.collection.mutable - /** * Gluten overwrite Delta: * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: + * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: * 1. In Clickhouse backend, we can't implement input_file_name() correctly, we can only implement * it so that it return a a list of filenames (concated by ','). */ case class MergeDataSizes( - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - rows: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - files: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - bytes: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - partitions: Option[Long] = None) + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + rows: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + files: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + bytes: Option[Long] = None, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + partitions: Option[Long] = None) /** * Represents the state of a single merge clause: - * - merge clause's (optional) predicate - * - action type (insert, update, delete) - * - action's expressions + * - merge clause's (optional) predicate + * - action type (insert, update, delete) + * - action's expressions */ -case class MergeClauseStats(condition: Option[String], actionType: String, actionExpr: Seq[String]) +case class MergeClauseStats( + condition: Option[String], + actionType: String, + actionExpr: Seq[String]) object MergeClauseStats { def apply(mergeClause: DeltaMergeIntoClause): MergeClauseStats = { @@ -93,9 +96,10 @@ case class MergeStats( insertExprs: Seq[String], deleteConditionExpr: String, - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED + // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/NOT MATCHED BY SOURCE matchedStats: Seq[MergeClauseStats], notMatchedStats: Seq[MergeClauseStats], + notMatchedBySourceStats: Seq[MergeClauseStats], // Timings executionTimeMs: Long, @@ -126,8 +130,12 @@ case class MergeStats( targetPartitionsAddedTo: Option[Long], targetRowsCopied: Long, targetRowsUpdated: Long, + targetRowsMatchedUpdated: Long, + targetRowsNotMatchedBySourceUpdated: Long, targetRowsInserted: Long, targetRowsDeleted: Long, + targetRowsMatchedDeleted: Long, + targetRowsNotMatchedBySourceDeleted: Long, // MergeMaterializeSource stats materializeSourceReason: Option[String] = None, @@ -142,6 +150,7 @@ object MergeStats { condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], + notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], isPartitioned: Boolean): MergeStats = { def metricValueIfPartitioned(metricName: String): Option[Long] = { @@ -152,9 +161,11 @@ object MergeStats { // Merge condition expression conditionExpr = condition.sql, - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED + // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED/ + // NOT MATCHED BY SOURCE matchedStats = matchedClauses.map(MergeClauseStats(_)), notMatchedStats = notMatchedClauses.map(MergeClauseStats(_)), + notMatchedBySourceStats = notMatchedBySourceClauses.map(MergeClauseStats(_)), // Timings executionTimeMs = metrics("executionTimeMs").value, @@ -163,15 +174,17 @@ object MergeStats { // Data sizes of source and target at different stages of processing source = MergeDataSizes(rows = Some(metrics("numSourceRows").value)), - targetBeforeSkipping = MergeDataSizes( - files = Some(metrics("numTargetFilesBeforeSkipping").value), - bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), - targetAfterSkipping = MergeDataSizes( - files = Some(metrics("numTargetFilesAfterSkipping").value), - bytes = Some(metrics("numTargetBytesAfterSkipping").value), - partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping") - ), - sourceRowsInSecondScan = metrics.get("numSourceRowsInSecondScan").map(_.value).filter(_ >= 0), + targetBeforeSkipping = + MergeDataSizes( + files = Some(metrics("numTargetFilesBeforeSkipping").value), + bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), + targetAfterSkipping = + MergeDataSizes( + files = Some(metrics("numTargetFilesAfterSkipping").value), + bytes = Some(metrics("numTargetBytesAfterSkipping").value), + partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping")), + sourceRowsInSecondScan = + metrics.get("numSourceRowsInSecondScan").map(_.value).filter(_ >= 0), // Data change sizes targetFilesAdded = metrics("numTargetFilesAdded").value, @@ -184,49 +197,48 @@ object MergeStats { targetPartitionsAddedTo = metricValueIfPartitioned("numTargetPartitionsAddedTo"), targetRowsCopied = metrics("numTargetRowsCopied").value, targetRowsUpdated = metrics("numTargetRowsUpdated").value, + targetRowsMatchedUpdated = metrics("numTargetRowsMatchedUpdated").value, + targetRowsNotMatchedBySourceUpdated = metrics("numTargetRowsNotMatchedBySourceUpdated").value, targetRowsInserted = metrics("numTargetRowsInserted").value, targetRowsDeleted = metrics("numTargetRowsDeleted").value, + targetRowsMatchedDeleted = metrics("numTargetRowsMatchedDeleted").value, + targetRowsNotMatchedBySourceDeleted = metrics("numTargetRowsNotMatchedBySourceDeleted").value, // Deprecated fields updateConditionExpr = null, updateExprs = null, insertConditionExpr = null, insertExprs = null, - deleteConditionExpr = null - ) + deleteConditionExpr = null) } } /** * Performs a merge of a source query/table into a Delta table. * - * Issues an error message when the ON search_condition of the MERGE statement can match a single - * row from the target table with multiple rows of the source table-reference. + * Issues an error message when the ON search_condition of the MERGE statement can match + * a single row from the target table with multiple rows of the source table-reference. * * Algorithm: * - * Phase 1: Find the input files in target that are touched by the rows that satisfy the condition - * and verify that no two source rows match with the same target row. This is implemented as an - * inner-join using the given condition. See [[findTouchedFiles]] for more details. + * Phase 1: Find the input files in target that are touched by the rows that satisfy + * the condition and verify that no two source rows match with the same target row. + * This is implemented as an inner-join using the given condition. See [[findTouchedFiles]] + * for more details. * * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. * * Phase 3: Use the Delta protocol to atomically remove the touched files and add the new files. * - * @param source - * Source data to merge from - * @param target - * Target table to merge into - * @param targetFileIndex - * TahoeFileIndex of the target table - * @param condition - * Condition for a source row to match with a target row - * @param matchedClauses - * All info related to matched clauses. - * @param notMatchedClauses - * All info related to not matched clause. - * @param migratedSchema - * The final schema of the target - may be changed by schema evolution. + * @param source Source data to merge from + * @param target Target table to merge into + * @param targetFileIndex TahoeFileIndex of the target table + * @param condition Condition for a source row to match with a target row + * @param matchedClauses All info related to matched clauses. + * @param notMatchedClauses All info related to not matched clauses. + * @param notMatchedBySourceClauses All info related to not matched by source clauses. + * @param migratedSchema The final schema of the target - may be changed by schema + * evolution. */ case class MergeIntoCommand( @transient source: LogicalPlan, @@ -235,18 +247,18 @@ case class MergeIntoCommand( condition: Expression, matchedClauses: Seq[DeltaMergeIntoMatchedClause], notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], - migratedSchema: Option[StructType]) - extends LeafRunnableCommand + notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], + migratedSchema: Option[StructType]) extends LeafRunnableCommand with DeltaCommand with PredicateHelper with AnalysisHelper with ImplicitMetadataOperation with MergeIntoMaterializeSource { - import org.apache.spark.sql.delta.commands.cdc.CDCReader._ - import MergeIntoCommand._ + import SQLMetrics._ + import org.apache.spark.sql.delta.commands.cdc.CDCReader._ override val canMergeSchema: Boolean = conf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE) override val canOverwriteSchema: Boolean = false @@ -255,20 +267,18 @@ case class MergeIntoCommand( AttributeReference("num_affected_rows", LongType)(), AttributeReference("num_updated_rows", LongType)(), AttributeReference("num_deleted_rows", LongType)(), - AttributeReference("num_inserted_rows", LongType)() - ) + AttributeReference("num_inserted_rows", LongType)()) @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() @transient private lazy val targetDeltaLog: DeltaLog = targetFileIndex.deltaLog - /** - * Map to get target output attributes by name. The case sensitivity of the map is set accordingly - * to Spark configuration. + * Map to get target output attributes by name. + * The case sensitivity of the map is set accordingly to Spark configuration. */ @transient private lazy val targetOutputAttributesMap: Map[String, Attribute] = { - val attrMap: Map[String, Attribute] = target.outputSet.view - .map(attr => attr.name -> attr) - .toMap + val attrMap: Map[String, Attribute] = target + .outputSet.view + .map(attr => attr.name -> attr).toMap if (conf.caseSensitiveAnalysis) { attrMap } else { @@ -277,10 +287,10 @@ case class MergeIntoCommand( } /** Whether this merge statement has only a single insert (NOT MATCHED) clause. */ - private def isSingleInsertOnly: Boolean = matchedClauses.isEmpty && notMatchedClauses.length == 1 - - /** Whether this merge statement has only MATCHED clauses. */ - private def isMatchedOnly: Boolean = notMatchedClauses.isEmpty && matchedClauses.nonEmpty + private def isSingleInsertOnly: Boolean = + matchedClauses.isEmpty && notMatchedBySourceClauses.isEmpty && notMatchedClauses.length == 1 + /** Whether this merge statement has no insert (NOT MATCHED) clause. */ + private def hasNoInserts: Boolean = notMatchedClauses.isEmpty // We over-count numTargetRowsDeleted when there are multiple matches; // this is the amount of the overcount, so we can subtract it to get a correct final metric. @@ -293,7 +303,15 @@ case class MergeIntoCommand( "numTargetRowsCopied" -> createMetric(sc, "number of target rows rewritten unmodified"), "numTargetRowsInserted" -> createMetric(sc, "number of inserted rows"), "numTargetRowsUpdated" -> createMetric(sc, "number of updated rows"), + "numTargetRowsMatchedUpdated" -> + createMetric(sc, "number of rows updated by a matched clause"), + "numTargetRowsNotMatchedBySourceUpdated" -> + createMetric(sc, "number of rows updated by a not matched by source clause"), "numTargetRowsDeleted" -> createMetric(sc, "number of deleted rows"), + "numTargetRowsMatchedDeleted" -> + createMetric(sc, "number of rows deleted by a matched clause"), + "numTargetRowsNotMatchedBySourceDeleted" -> + createMetric(sc, "number of rows deleted by a not matched by source clause"), "numTargetFilesBeforeSkipping" -> createMetric(sc, "number of target files before skipping"), "numTargetFilesAfterSkipping" -> createMetric(sc, "number of target files after skipping"), "numTargetFilesRemoved" -> createMetric(sc, "number of files removed to target"), @@ -317,8 +335,7 @@ case class MergeIntoCommand( "scanTimeMs" -> createTimingMetric(sc, "time taken to scan the files for matches"), "rewriteTimeMs" -> - createTimingMetric(sc, "time taken to rewrite the matched files") - ) + createTimingMetric(sc, "time taken to rewrite the matched files")) override def run(spark: SparkSession): Seq[Row] = { metrics("executionTimeMs").set(0) @@ -334,7 +351,7 @@ case class MergeIntoCommand( if (newNullColumn.isDefined) { throw new AnalysisException( s"""Cannot add column '${newNullColumn.get}' with type 'void'. Please explicitly specify a - |non-void type.""".stripMargin.replaceAll("\n", " ") + |non-void type.""".stripMargin.replaceAll("\n", " ") ) } } @@ -344,118 +361,107 @@ case class MergeIntoCommand( } else { // If it is determined that source should be materialized, wrap the execution with retries, // in case the data of the materialized source is lost. - runWithMaterializedSourceLostRetries(spark, targetFileIndex.deltaLog, metrics, runMerge) + runWithMaterializedSourceLostRetries( + spark, targetFileIndex.deltaLog, metrics, runMerge) } } protected def runMerge(spark: SparkSession): Seq[Row] = { recordDeltaOperation(targetDeltaLog, "delta.dml.merge") { val startTime = System.nanoTime() - targetDeltaLog.withNewTransaction { - deltaTxn => - if (target.schema.size != deltaTxn.metadata.schema.size) { - throw DeltaErrors.schemaChangedSinceAnalysis( - atAnalysis = target.schema, - latestSchema = deltaTxn.metadata.schema) - } + targetDeltaLog.withNewTransaction { deltaTxn => + if (hasBeenExecuted(deltaTxn, spark)) { + sendDriverMetrics(spark, metrics) + return Seq.empty + } + if (target.schema.size != deltaTxn.metadata.schema.size) { + throw DeltaErrors.schemaChangedSinceAnalysis( + atAnalysis = target.schema, latestSchema = deltaTxn.metadata.schema) + } - if (canMergeSchema) { - updateMetadata( - spark, - deltaTxn, - migratedSchema.getOrElse(target.schema), - deltaTxn.metadata.partitionColumns, - deltaTxn.metadata.configuration, - isOverwriteMode = false, - rearrangeOnly = false - ) - } + if (canMergeSchema) { + updateMetadata( + spark, deltaTxn, migratedSchema.getOrElse(target.schema), + deltaTxn.metadata.partitionColumns, deltaTxn.metadata.configuration, + isOverwriteMode = false, rearrangeOnly = false) + } - // If materialized, prepare the DF reading the materialize source - // Otherwise, prepare a regular DF from source plan. - val materializeSourceReason = prepareSourceDFAndReturnMaterializeReason( - spark, - source, - condition, - matchedClauses, - notMatchedClauses, - isSingleInsertOnly) - - val deltaActions = { - if (isSingleInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { - writeInsertsOnlyWhenNoMatchedClauses(spark, deltaTxn) - } else { - val filesToRewrite = findTouchedFiles(spark, deltaTxn) - val newWrittenFiles = withStatusCode("DELTA", "Writing merged data") { - writeAllChanges(spark, deltaTxn, filesToRewrite) - } - filesToRewrite.map(_.remove) ++ newWrittenFiles + // If materialized, prepare the DF reading the materialize source + // Otherwise, prepare a regular DF from source plan. + val materializeSourceReason = prepareSourceDFAndReturnMaterializeReason( + spark, + source, + condition, + matchedClauses, + notMatchedClauses, + isSingleInsertOnly) + + val deltaActions = { + if (isSingleInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { + writeInsertsOnlyWhenNoMatchedClauses(spark, deltaTxn) + } else { + val filesToRewrite = findTouchedFiles(spark, deltaTxn) + val newWrittenFiles = withStatusCode("DELTA", "Writing merged data") { + writeAllChanges(spark, deltaTxn, filesToRewrite) } + filesToRewrite.map(_.remove) ++ newWrittenFiles } + } - // Metrics should be recorded before commit (where they are written to delta logs). - metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) - deltaTxn.registerSQLMetrics(spark, metrics) - - // This is a best-effort sanity check. - if ( - metrics("numSourceRowsInSecondScan").value >= 0 && - metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value - ) { - log.warn( - s"Merge source has ${metrics("numSourceRows")} rows in initial scan but " + - s"${metrics("numSourceRowsInSecondScan")} rows in second scan") - if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { - throw DeltaErrors.sourceNotDeterministicInMergeException(spark) - } + val finalActions = createSetTransaction(spark, targetDeltaLog).toSeq ++ deltaActions + // Metrics should be recorded before commit (where they are written to delta logs). + metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) + deltaTxn.registerSQLMetrics(spark, metrics) + + // This is a best-effort sanity check. + if (metrics("numSourceRowsInSecondScan").value >= 0 && + metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value) { + log.warn(s"Merge source has ${metrics("numSourceRows")} rows in initial scan but " + + s"${metrics("numSourceRowsInSecondScan")} rows in second scan") + if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { + throw DeltaErrors.sourceNotDeterministicInMergeException(spark) } + } - deltaTxn.commit( - deltaActions, - DeltaOperations.Merge( - Option(condition.sql), - matchedClauses.map(DeltaOperations.MergePredicate(_)), - notMatchedClauses.map(DeltaOperations.MergePredicate(_))) - ) - - // Record metrics - var stats = MergeStats.fromMergeSQLMetrics( - metrics, - condition, - matchedClauses, - notMatchedClauses, - deltaTxn.metadata.partitionColumns.nonEmpty) - stats = stats.copy( - materializeSourceReason = Some(materializeSourceReason.toString), - materializeSourceAttempts = Some(attempt)) - - recordDeltaEvent(targetFileIndex.deltaLog, "delta.dml.merge.stats", data = stats) + deltaTxn.commitIfNeeded( + finalActions, + DeltaOperations.Merge( + Option(condition.sql), + matchedClauses.map(DeltaOperations.MergePredicate(_)), + notMatchedClauses.map(DeltaOperations.MergePredicate(_)), + notMatchedBySourceClauses.map(DeltaOperations.MergePredicate(_)))) + + // Record metrics + var stats = MergeStats.fromMergeSQLMetrics( + metrics, + condition, + matchedClauses, + notMatchedClauses, + notMatchedBySourceClauses, + deltaTxn.metadata.partitionColumns.nonEmpty) + stats = stats.copy( + materializeSourceReason = Some(materializeSourceReason.toString), + materializeSourceAttempts = Some(attempt)) + + recordDeltaEvent(targetFileIndex.deltaLog, "delta.dml.merge.stats", data = stats) } spark.sharedState.cacheManager.recacheByPlan(spark, target) } - // This is needed to make the SQL metrics visible in the Spark UI. Also this needs - // to be outside the recordMergeOperation because this method will update some metric. - val executionId = spark.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates(spark.sparkContext, executionId, metrics.values.toSeq) - Seq( - Row( - metrics("numTargetRowsUpdated").value + metrics("numTargetRowsDeleted").value + - metrics("numTargetRowsInserted").value, - metrics("numTargetRowsUpdated").value, - metrics("numTargetRowsDeleted").value, - metrics("numTargetRowsInserted").value - )) + sendDriverMetrics(spark, metrics) + Seq(Row(metrics("numTargetRowsUpdated").value + metrics("numTargetRowsDeleted").value + + metrics("numTargetRowsInserted").value, metrics("numTargetRowsUpdated").value, + metrics("numTargetRowsDeleted").value, metrics("numTargetRowsInserted").value)) } /** * Find the target table files that contain the rows that satisfy the merge condition. This is - * implemented as an inner-join between the source query/table and the target table using the - * merge condition. + * implemented as an inner-join between the source query/table and the target table using + * the merge condition. */ private def findTouchedFiles( - spark: SparkSession, - deltaTxn: OptimisticTransaction + spark: SparkSession, + deltaTxn: OptimisticTransaction ): Seq[AddFile] = recordMergeOperation(sqlMetricName = "scanTimeMs") { // Accumulator to collect all the distinct touched files @@ -463,34 +469,40 @@ case class MergeIntoCommand( spark.sparkContext.register(touchedFilesAccum, TOUCHED_FILES_ACCUM_NAME) // UDFs to records touched files names and add them to the accumulator - val recordTouchedFileName = DeltaUDF - .intFromString { - fileName => - fileName.split(",").foreach(name => touchedFilesAccum.add(name)) - 1 + val recordTouchedFileName = DeltaUDF.intFromString { fileName => + // --- modified start + fileName.split(",").foreach(name => touchedFilesAccum.add(name)) + // --- modified end + 1 + }.asNondeterministic() + + // Prune non-matching files if we don't need to collect them for NOT MATCHED BY SOURCE clauses. + val dataSkippedFiles = + if (notMatchedBySourceClauses.isEmpty) { + val targetOnlyPredicates = + splitConjunctivePredicates(condition).filter(_.references.subsetOf(target.outputSet)) + deltaTxn.filterFiles(targetOnlyPredicates) + } else { + deltaTxn.filterFiles() } - .asNondeterministic() - - // Skip data based on the merge condition - val targetOnlyPredicates = - splitConjunctivePredicates(condition).filter(_.references.subsetOf(target.outputSet)) - val dataSkippedFiles = deltaTxn.filterFiles(targetOnlyPredicates) // UDF to increment metrics val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRows") val sourceDF = getSourceDF() .filter(new Column(incrSourceRowCountExpr)) - // Apply inner join to between source and target using the merge condition to find matches + // Join the source and target table using the merge condition to find touched files. An inner + // join collects all candidate files for MATCHED clauses, a right outer join also includes + // candidates for NOT MATCHED BY SOURCE clauses. // In addition, we attach two columns // - a monotonically increasing row id for target rows to later identify whether the same // target row is modified by multiple user or not // - the target file name the row is from to later identify the files touched by matched rows - val targetDF = Dataset - .ofRows(spark, buildTargetPlanWithFiles(deltaTxn, dataSkippedFiles)) + val joinType = if (notMatchedBySourceClauses.isEmpty) "inner" else "right_outer" + val targetDF = buildTargetPlanWithFiles(spark, deltaTxn, dataSkippedFiles) .withColumn(ROW_ID_COL, monotonically_increasing_id()) .withColumn(FILE_NAME_COL, input_file_name()) - val joinToFindTouchedFiles = sourceDF.join(targetDF, new Column(condition), "inner") + val joinToFindTouchedFiles = sourceDF.join(targetDF, new Column(condition), joinType) // Process the matches from the inner join to record touched files and find multiple matches val collectTouchedFiles = joinToFindTouchedFiles @@ -542,16 +554,14 @@ case class MergeIntoCommand( logTrace(s"findTouchedFiles: matched files:\n\t${touchedFileNames.mkString("\n\t")}") val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, dataSkippedFiles) - val touchedAddFiles = - touchedFileNames.map(f => getTouchedFile(targetDeltaLog.dataPath, f, nameToAddFileMap)) + val touchedAddFiles = touchedFileNames.map(f => + getTouchedFile(targetDeltaLog.dataPath, f, nameToAddFileMap)) // When the target table is empty, and the optimizer optimized away the join entirely // numSourceRows will be incorrectly 0. We need to scan the source table once to get the correct // metric here. - if ( - metrics("numSourceRows").value == 0 && - (dataSkippedFiles.isEmpty || targetDF.take(1).isEmpty) - ) { + if (metrics("numSourceRows").value == 0 && + (dataSkippedFiles.isEmpty || targetDF.take(1).isEmpty)) { val numSourceRows = sourceDF.count() metrics("numSourceRows").set(numSourceRows) } @@ -572,15 +582,15 @@ case class MergeIntoCommand( } /** - * This is an optimization of the case when there is no update clause for the merge. We perform an - * left anti join on the source data to find the rows to be inserted. + * This is an optimization of the case when there is no update clause for the merge. + * We perform an left anti join on the source data to find the rows to be inserted. * * This will currently only optimize for the case when there is a _single_ notMatchedClause. */ private def writeInsertsOnlyWhenNoMatchedClauses( spark: SparkSession, deltaTxn: OptimisticTransaction - ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { + ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { // UDFs to update metrics val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRows") @@ -589,9 +599,8 @@ case class MergeIntoCommand( val outputColNames = getTargetOutputCols(deltaTxn).map(_.name) // we use head here since we know there is only a single notMatchedClause val outputExprs = notMatchedClauses.head.resolvedActions.map(_.expr) - val outputCols = outputExprs.zip(outputColNames).map { - case (expr, name) => - new Column(Alias(expr, name)()) + val outputCols = outputExprs.zip(outputColNames).map { case (expr, name) => + new Column(Alias(expr, name)()) } // source DataFrame @@ -606,10 +615,9 @@ case class MergeIntoCommand( val dataSkippedFiles = deltaTxn.filterFiles(targetOnlyPredicates) // target DataFrame - val targetDF = Dataset.ofRows(spark, buildTargetPlanWithFiles(deltaTxn, dataSkippedFiles)) + val targetDF = buildTargetPlanWithFiles(spark, deltaTxn, dataSkippedFiles) - val insertDf = sourceDF - .join(targetDF, new Column(condition), "leftanti") + val insertDf = sourceDF.join(targetDF, new Column(condition), "leftanti") .select(outputCols: _*) .filter(new Column(incrInsertedCountExpr)) @@ -652,11 +660,11 @@ case class MergeIntoCommand( * CDC_TYPE_COL_NAME used for handling CDC when enabled. */ private def writeAllChanges( - spark: SparkSession, - deltaTxn: OptimisticTransaction, - filesToRewrite: Seq[AddFile] + spark: SparkSession, + deltaTxn: OptimisticTransaction, + filesToRewrite: Seq[AddFile] ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { - import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral} + import org.apache.spark.sql.catalyst.expressions.Literal.{TrueLiteral, FalseLiteral} val cdcEnabled = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(deltaTxn.metadata) @@ -685,32 +693,35 @@ case class MergeIntoCommand( // need to drop the duplicate matches. val isDeleteWithDuplicateMatchesAndCdc = multipleMatchDeleteOnlyOvercount.nonEmpty && cdcEnabled - // Generate a new logical plan that has same output attributes exprIds as the target plan. + // Generate a new target dataframe that has same output attributes exprIds as the target plan. // This allows us to apply the existing resolved update/insert expressions. - val newTarget = buildTargetPlanWithFiles(deltaTxn, filesToRewrite) - val joinType = - if ( - isMatchedOnly && - spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED) - ) { - "rightOuter" - } else { - "fullOuter" - } + val baseTargetDF = buildTargetPlanWithFiles(spark, deltaTxn, filesToRewrite) + val joinType = if (hasNoInserts && + spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED)) { + "rightOuter" + } else { + "fullOuter" + } logDebug(s"""writeAllChanges using $joinType join: | source.output: ${source.outputSet} | target.output: ${target.outputSet} | condition: $condition - | newTarget.output: ${newTarget.outputSet} + | newTarget.output: ${baseTargetDF.queryExecution.logical.outputSet} """.stripMargin) // UDFs to update metrics val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRowsInSecondScan") val incrUpdatedCountExpr = makeMetricUpdateUDF("numTargetRowsUpdated") + val incrUpdatedMatchedCountExpr = makeMetricUpdateUDF("numTargetRowsMatchedUpdated") + val incrUpdatedNotMatchedBySourceCountExpr = + makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceUpdated") val incrInsertedCountExpr = makeMetricUpdateUDF("numTargetRowsInserted") val incrNoopCountExpr = makeMetricUpdateUDF("numTargetRowsCopied") val incrDeletedCountExpr = makeMetricUpdateUDF("numTargetRowsDeleted") + val incrDeletedMatchedCountExpr = makeMetricUpdateUDF("numTargetRowsMatchedDeleted") + val incrDeletedNotMatchedBySourceCountExpr = + makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceDeleted") // Apply an outer join to find both, matches and non-matches. We are adding two boolean fields // with value `true`, one to each side of the join. Whether this field is null or not after @@ -721,8 +732,7 @@ case class MergeIntoCommand( // insert clause. See above at isDeleteWithDuplicateMatchesAndCdc definition for more details. var sourceDF = getSourceDF() .withColumn(SOURCE_ROW_PRESENT_COL, new Column(incrSourceRowCountExpr)) - var targetDF = Dataset - .ofRows(spark, newTarget) + var targetDF = baseTargetDF .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) if (isDeleteWithDuplicateMatchesAndCdc) { targetDF = targetDF.withColumn(TARGET_ROW_ID_COL, monotonically_increasing_id()) @@ -783,12 +793,13 @@ case class MergeIntoCommand( .add(CDC_TYPE_COLUMN_NAME, DataTypes.StringType) } - def updateOutput(resolvedActions: Seq[DeltaMergeAction]): Seq[Seq[Expression]] = { + def updateOutput(resolvedActions: Seq[DeltaMergeAction], incrMetricExpr: Expression) + : Seq[Seq[Expression]] = { val updateExprs = { // Generate update expressions and set ROW_DELETED_COL = false and // CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC val mainDataOutput = resolvedActions.map(_.expr) :+ FalseLiteral :+ - incrUpdatedCountExpr :+ CDC_TYPE_NOT_CDC + incrMetricExpr :+ CDC_TYPE_NOT_CDC if (cdcEnabled) { // For update preimage, we have do a no-op copy with ROW_DELETED_COL = false and // CDC_TYPE_COLUMN_NAME = CDC_TYPE_UPDATE_PREIMAGE and INCR_ROW_COUNT_COL as a no-op @@ -808,11 +819,11 @@ case class MergeIntoCommand( updateExprs.map(resolveOnJoinedPlan) } - def deleteOutput(): Seq[Seq[Expression]] = { + def deleteOutput(incrMetricExpr: Expression): Seq[Seq[Expression]] = { val deleteExprs = { // Generate expressions to set the ROW_DELETED_COL = true and CDC_TYPE_COLUMN_NAME = // CDC_TYPE_NOT_CDC - val mainDataOutput = targetOutputCols :+ TrueLiteral :+ incrDeletedCountExpr :+ + val mainDataOutput = targetOutputCols :+ TrueLiteral :+ incrMetricExpr :+ CDC_TYPE_NOT_CDC if (cdcEnabled) { // For delete we do a no-op copy with ROW_DELETED_COL = false, INCR_ROW_COUNT_COL as a @@ -827,7 +838,8 @@ case class MergeIntoCommand( deleteExprs.map(resolveOnJoinedPlan) } - def insertOutput(resolvedActions: Seq[DeltaMergeAction]): Seq[Seq[Expression]] = { + def insertOutput(resolvedActions: Seq[DeltaMergeAction], incrMetricExpr: Expression) + : Seq[Seq[Expression]] = { // Generate insert expressions and set ROW_DELETED_COL = false and // CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC val insertExprs = resolvedActions.map(_.expr) @@ -839,9 +851,9 @@ case class MergeIntoCommand( // isDeleteWithDuplicateMatchesAndCdc definition for more details. insertExprs :+ Alias(Literal(null), TARGET_ROW_ID_COL)() :+ UnresolvedAttribute(SOURCE_ROW_ID_COL) :+ - FalseLiteral :+ incrInsertedCountExpr :+ CDC_TYPE_NOT_CDC + FalseLiteral :+ incrMetricExpr :+ CDC_TYPE_NOT_CDC } else { - insertExprs :+ FalseLiteral :+ incrInsertedCountExpr :+ CDC_TYPE_NOT_CDC + insertExprs :+ FalseLiteral :+ incrMetricExpr :+ CDC_TYPE_NOT_CDC } ) if (cdcEnabled) { @@ -856,9 +868,18 @@ case class MergeIntoCommand( } def clauseOutput(clause: DeltaMergeIntoClause): Seq[Seq[Expression]] = clause match { - case u: DeltaMergeIntoMatchedUpdateClause => updateOutput(u.resolvedActions) - case _: DeltaMergeIntoMatchedDeleteClause => deleteOutput() - case i: DeltaMergeIntoNotMatchedInsertClause => insertOutput(i.resolvedActions) + case u: DeltaMergeIntoMatchedUpdateClause => + updateOutput(u.resolvedActions, And(incrUpdatedCountExpr, incrUpdatedMatchedCountExpr)) + case _: DeltaMergeIntoMatchedDeleteClause => + deleteOutput(And(incrDeletedCountExpr, incrDeletedMatchedCountExpr)) + case i: DeltaMergeIntoNotMatchedInsertClause => + insertOutput(i.resolvedActions, incrInsertedCountExpr) + case u: DeltaMergeIntoNotMatchedBySourceUpdateClause => + updateOutput( + u.resolvedActions, + And(incrUpdatedCountExpr, incrUpdatedNotMatchedBySourceCountExpr)) + case _: DeltaMergeIntoNotMatchedBySourceDeleteClause => + deleteOutput(And(incrDeletedCountExpr, incrDeletedNotMatchedBySourceCountExpr)) } def clauseCondition(clause: DeltaMergeIntoClause): Expression = { @@ -877,15 +898,16 @@ case class MergeIntoCommand( matchedOutputs = matchedClauses.map(clauseOutput), notMatchedConditions = notMatchedClauses.map(clauseCondition), notMatchedOutputs = notMatchedClauses.map(clauseOutput), - noopCopyOutput = resolveOnJoinedPlan( - targetOutputCols :+ FalseLiteral :+ incrNoopCountExpr :+ + notMatchedBySourceConditions = notMatchedBySourceClauses.map(clauseCondition), + notMatchedBySourceOutputs = notMatchedBySourceClauses.map(clauseOutput), + noopCopyOutput = + resolveOnJoinedPlan(targetOutputCols :+ FalseLiteral :+ incrNoopCountExpr :+ CDC_TYPE_NOT_CDC), deleteRowOutput = resolveOnJoinedPlan(targetOutputCols :+ TrueLiteral :+ TrueLiteral :+ CDC_TYPE_NOT_CDC), joinedAttributes = joinedPlan.output, joinedRowEncoder = joinedRowEncoder, - outputRowEncoder = outputRowEncoder - ) + outputRowEncoder = outputRowEncoder) var outputDF = Dataset.ofRows(spark, joinedPlan).mapPartitions(processor.processPartition)(outputRowEncoder) @@ -922,7 +944,7 @@ case class MergeIntoCommand( val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) - metrics("numTargetChangeFileBytes") += newFiles.collect { case f: AddCDCFile => f.size }.sum + metrics("numTargetChangeFileBytes") += newFiles.collect{ case f: AddCDCFile => f.size }.sum metrics("numTargetBytesAdded") += addedBytes metrics("numTargetPartitionsAddedTo") += addedPartitions if (multipleMatchDeleteOnlyOvercount.isDefined) { @@ -931,24 +953,29 @@ case class MergeIntoCommand( metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get assert(actualRowsDeleted >= 0) metrics("numTargetRowsDeleted").set(actualRowsDeleted) + val actualRowsMatchedDeleted = + metrics("numTargetRowsMatchedDeleted").value - multipleMatchDeleteOnlyOvercount.get + assert(actualRowsMatchedDeleted >= 0) + metrics("numTargetRowsMatchedDeleted").set(actualRowsMatchedDeleted) } newFiles } + /** - * Build a new logical plan using the given `files` that has the same output columns (exprIds) as - * the `target` logical plan, so that existing update/insert expressions can be applied on this - * new plan. + * Build a new logical plan using the given `files` that has the same output columns (exprIds) + * as the `target` logical plan, so that existing update/insert expressions can be applied + * on this new plan. */ private def buildTargetPlanWithFiles( - deltaTxn: OptimisticTransaction, - files: Seq[AddFile]): LogicalPlan = { + spark: SparkSession, + deltaTxn: OptimisticTransaction, + files: Seq[AddFile]): DataFrame = { val targetOutputCols = getTargetOutputCols(deltaTxn) val targetOutputColsMap = { val colsMap: Map[String, NamedExpression] = targetOutputCols.view - .map(col => col.name -> col) - .toMap + .map(col => col.name -> col).toMap if (conf.caseSensitiveAnalysis) { colsMap } else { @@ -968,8 +995,7 @@ case class MergeIntoCommand( // We can ignore the new columns which aren't yet AttributeReferences. targetOutputCols.collect { case a: AttributeReference => a }, catalogTbl, - isStreaming - ) + isStreaming) } // In case of schema evolution & column mapping, we would also need to rebuild the file format @@ -987,14 +1013,11 @@ case class MergeIntoCommand( // create an alias val aliases = plan.output.map { case newAttrib: AttributeReference => - val existingTargetAttrib = targetOutputColsMap - .get(newAttrib.name) + val existingTargetAttrib = targetOutputColsMap.get(newAttrib.name) .getOrElse { throw DeltaErrors.failedFindAttributeInOutputColumns( - newAttrib.name, - targetOutputCols.mkString(",")) - } - .asInstanceOf[AttributeReference] + newAttrib.name, targetOutputCols.mkString(",")) + }.asInstanceOf[AttributeReference] if (existingTargetAttrib.exprId == newAttrib.exprId) { // It's not valid to alias an expression to its own exprId (this is considered a @@ -1005,7 +1028,7 @@ case class MergeIntoCommand( } } - Project(aliases, plan) + Dataset.ofRows(spark, Project(aliases, plan)) } /** Expressions to increment SQL metrics */ @@ -1016,18 +1039,20 @@ case class MergeIntoCommand( } private def getTargetOutputCols(txn: OptimisticTransaction): Seq[NamedExpression] = { - txn.metadata.schema.map { - col => - targetOutputAttributesMap - .get(col.name) - .map(a => AttributeReference(col.name, col.dataType, col.nullable)(a.exprId)) - .getOrElse(Alias(Literal(null), col.name)()) + txn.metadata.schema.map { col => + targetOutputAttributesMap + .get(col.name) + .map { a => + AttributeReference(col.name, col.dataType, col.nullable)(a.exprId) + } + .getOrElse(Alias(Literal(null), col.name)() + ) } } /** - * Repartitions the output DataFrame by the partition columns if table is partitioned and - * `merge.repartitionBeforeWrite.enabled` is set to true. + * Repartitions the output DataFrame by the partition columns if table is partitioned + * and `merge.repartitionBeforeWrite.enabled` is set to true. */ protected def repartitionIfNeeded( spark: SparkSession, @@ -1043,10 +1068,8 @@ case class MergeIntoCommand( /** * Execute the given `thunk` and return its result while recording the time taken to do it. * - * @param sqlMetricName - * name of SQL metric to update with the time taken by the thunk - * @param thunk - * the code to execute + * @param sqlMetricName name of SQL metric to update with the time taken by the thunk + * @param thunk the code to execute */ private def recordMergeOperation[A](sqlMetricName: String = null)(thunk: => A): A = { val startTimeNs = System.nanoTime() @@ -1060,7 +1083,6 @@ case class MergeIntoCommand( } object MergeIntoCommand { - /** * Spark UI will track all normal accumulators along with Spark tasks to show them on Web UI. * However, the accumulator used by `MergeIntoCommand` can store a very large value since it @@ -1081,31 +1103,29 @@ object MergeIntoCommand { val INCR_ROW_COUNT_COL = "_incr_row_count_" /** - * @param targetRowHasNoMatch - * whether a joined row is a target row with no match in the source table - * @param sourceRowHasNoMatch - * whether a joined row is a source row with no match in the target table - * @param matchedConditions - * condition for each match clause - * @param matchedOutputs - * corresponding output for each match clause. for each clause, we have 1-3 output rows, each of - * which is a sequence of expressions to apply to the joined row - * @param notMatchedConditions - * condition for each not-matched clause - * @param notMatchedOutputs - * corresponding output for each not-matched clause. for each clause, we have 1-2 output rows, - * each of which is a sequence of expressions to apply to the joined row - * @param noopCopyOutput - * no-op expression to copy a target row to the output - * @param deleteRowOutput - * expression to drop a row from the final output. this is used for source rows that don't match - * any not-matched clauses - * @param joinedAttributes - * schema of our outer-joined dataframe - * @param joinedRowEncoder - * joinedDF row encoder - * @param outputRowEncoder - * final output row encoder + * @param targetRowHasNoMatch whether a joined row is a target row with no match in the source + * table + * @param sourceRowHasNoMatch whether a joined row is a source row with no match in the target + * table + * @param matchedConditions condition for each match clause + * @param matchedOutputs corresponding output for each match clause. for each clause, we + * have 1-3 output rows, each of which is a sequence of expressions + * to apply to the joined row + * @param notMatchedConditions condition for each not-matched clause + * @param notMatchedOutputs corresponding output for each not-matched clause. for each clause, + * we have 1-2 output rows, each of which is a sequence of + * expressions to apply to the joined row + * @param notMatchedBySourceConditions condition for each not-matched-by-source clause + * @param notMatchedBySourceOutputs corresponding output for each not-matched-by-source + * clause. for each clause, we have 1-3 output rows, each of + * which is a sequence of expressions to apply to the joined + * row + * @param noopCopyOutput no-op expression to copy a target row to the output + * @param deleteRowOutput expression to drop a row from the final output. this is used for + * source rows that don't match any not-matched clauses + * @param joinedAttributes schema of our outer-joined dataframe + * @param joinedRowEncoder joinedDF row encoder + * @param outputRowEncoder final output row encoder */ class JoinedRowProcessor( targetRowHasNoMatch: Expression, @@ -1114,12 +1134,13 @@ object MergeIntoCommand { matchedOutputs: Seq[Seq[Seq[Expression]]], notMatchedConditions: Seq[Expression], notMatchedOutputs: Seq[Seq[Seq[Expression]]], + notMatchedBySourceConditions: Seq[Expression], + notMatchedBySourceOutputs: Seq[Seq[Seq[Expression]]], noopCopyOutput: Seq[Expression], deleteRowOutput: Seq[Expression], joinedAttributes: Seq[Attribute], joinedRowEncoder: ExpressionEncoder[Row], - outputRowEncoder: ExpressionEncoder[Row]) - extends Serializable { + outputRowEncoder: ExpressionEncoder[Row]) extends Serializable { private def generateProjection(exprs: Seq[Expression]): UnsafeProjection = { UnsafeProjection.create(exprs, joinedAttributes) @@ -1137,6 +1158,8 @@ object MergeIntoCommand { val matchedProjs = matchedOutputs.map(_.map(generateProjection)) val notMatchedPreds = notMatchedConditions.map(generatePredicate) val notMatchedProjs = notMatchedOutputs.map(_.map(generateProjection)) + val notMatchedBySourcePreds = notMatchedBySourceConditions.map(generatePredicate) + val notMatchedBySourceProjs = notMatchedBySourceOutputs.map(_.map(generateProjection)) val noopCopyProj = generateProjection(noopCopyOutput) val deleteRowProj = generateProjection(deleteRowOutput) val outputProj = UnsafeProjection.create(outputRowEncoder.schema) @@ -1145,35 +1168,33 @@ object MergeIntoCommand { // then CDC must be disabled and it's the column after our output cols def shouldDeleteRow(row: InternalRow): Boolean = { row.getBoolean( - outputRowEncoder.schema - .getFieldIndex(ROW_DROPPED_COL) + outputRowEncoder.schema.getFieldIndex(ROW_DROPPED_COL) .getOrElse(outputRowEncoder.schema.fields.size) ) } def processRow(inputRow: InternalRow): Iterator[InternalRow] = { - if (targetRowHasNoMatchPred.eval(inputRow)) { - // Target row did not match any source row, so just copy it to the output - Iterator(noopCopyProj.apply(inputRow)) + // Identify which set of clauses to execute: matched, not-matched or not-matched-by-source + val (predicates, projections, noopAction) = if (targetRowHasNoMatchPred.eval(inputRow)) { + // Target row did not match any source row, so update the target row. + (notMatchedBySourcePreds, notMatchedBySourceProjs, noopCopyProj) + } else if (sourceRowHasNoMatchPred.eval(inputRow)) { + // Source row did not match with any target row, so insert the new source row + (notMatchedPreds, notMatchedProjs, deleteRowProj) } else { - // identify which set of clauses to execute: matched or not-matched ones - val (predicates, projections, noopAction) = if (sourceRowHasNoMatchPred.eval(inputRow)) { - // Source row did not match with any target row, so insert the new source row - (notMatchedPreds, notMatchedProjs, deleteRowProj) - } else { - // Source row matched with target row, so update the target row - (matchedPreds, matchedProjs, noopCopyProj) - } + // Source row matched with target row, so update the target row + (matchedPreds, matchedProjs, noopCopyProj) + } - // find (predicate, projection) pair whose predicate satisfies inputRow - val pair = - (predicates.zip(projections)).find { case (predicate, _) => predicate.eval(inputRow) } + // find (predicate, projection) pair whose predicate satisfies inputRow + val pair = (predicates zip projections).find { + case (predicate, _) => predicate.eval(inputRow) + } - pair match { - case Some((_, projections)) => - projections.map(_.apply(inputRow)).iterator - case None => Iterator(noopAction.apply(inputRow)) - } + pair match { + case Some((_, projections)) => + projections.map(_.apply(inputRow)).iterator + case None => Iterator(noopAction.apply(inputRow)) } } @@ -1183,7 +1204,9 @@ object MergeIntoCommand { .map(toRow) .flatMap(processRow) .filter(!shouldDeleteRow(_)) - .map(notDeletedInternalRow => fromRow(outputProj(notDeletedInternalRow))) + .map { notDeletedInternalRow => + fromRow(outputProj(notDeletedInternalRow)) + } } } diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 51% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index e69b3aa65931..7fa2c97d9006 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -14,24 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta.commands -import org.apache.spark.SparkContext -import org.apache.spark.SparkContext.SPARK_JOB_GROUP_ID -import org.apache.spark.sql.{AnalysisException, Encoders, Row, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} +import java.util.ConcurrentModificationException + +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.sql.delta.skipping.MultiDimClustering import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.DeltaOperations.Operation -import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, RemoveFile} +import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} import org.apache.spark.sql.delta.commands.OptimizeTableCommandOverwrites.{getDeltaLogClickhouse, groupFilesIntoBinsClickhouse, runOptimizeBinJobClickhouse} import org.apache.spark.sql.delta.commands.optimize._ import org.apache.spark.sql.delta.files.SQLMetricsReporting import org.apache.spark.sql.delta.schema.SchemaUtils -import org.apache.spark.sql.delta.skipping.MultiDimClustering import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkContext +import org.apache.spark.SparkContext.SPARK_JOB_GROUP_ID +import org.apache.spark.sql.{AnalysisException, Encoders, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils import org.apache.spark.sql.execution.metric.SQLMetric @@ -39,15 +46,13 @@ import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.types._ import org.apache.spark.util.{SystemClock, ThreadUtils} -import java.util.ConcurrentModificationException - -import scala.collection.mutable.ArrayBuffer - /** * Gluten overwrite Delta: * - * This file is copied from Delta 2.2.0. It is modified in: - * 1. getDeltaLogClickhouse 2. runOptimizeBinJobClickhouse 3. groupFilesIntoBinsClickhouse + * This file is copied from Delta 2.3.0. It is modified in: + * 1. getDeltaLogClickhouse + * 2. runOptimizeBinJobClickhouse + * 3. groupFilesIntoBinsClickhouse */ /** Base class defining abstract optimize command */ @@ -59,16 +64,12 @@ abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaComman /** * Validates ZOrderBy columns - * - validates that partitions columns are not used in `unresolvedZOrderByCols` - * - validates that we already collect stats for all the columns used in - * `unresolvedZOrderByCols` + * - validates that partitions columns are not used in `unresolvedZOrderByCols` + * - validates that we already collect stats for all the columns used in `unresolvedZOrderByCols` * - * @param spark - * [[SparkSession]] to use - * @param txn - * the [[OptimisticTransaction]] being used to optimize - * @param unresolvedZOrderByCols - * Seq of [[UnresolvedAttribute]] corresponding to zOrderBy columns + * @param spark [[SparkSession]] to use + * @param txn the [[OptimisticTransaction]] being used to optimize + * @param unresolvedZOrderByCols Seq of [[UnresolvedAttribute]] corresponding to zOrderBy columns */ def validateZorderByColumns( spark: SparkSession, @@ -80,32 +81,32 @@ abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaComman val dataSchema = StructType(metadata.schema.filterNot(c => partitionColumns.contains(c.name))) val df = spark.createDataFrame(new java.util.ArrayList[Row](), dataSchema) - val checkColStat = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK) + val checkColStat = spark.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK) val statCollectionSchema = txn.snapshot.statCollectionSchema val colsWithoutStats = ArrayBuffer[String]() - unresolvedZOrderByCols.foreach { - colAttribute => - val colName = colAttribute.name - if (checkColStat) { - try { - SchemaUtils.findColumnPosition(colAttribute.nameParts, statCollectionSchema) - } catch { - case e: AnalysisException if e.getMessage.contains("Couldn't find column") => - colsWithoutStats.append(colName) - } - } - val isNameEqual = spark.sessionState.conf.resolver - if (partitionColumns.find(isNameEqual(_, colName)).nonEmpty) { - throw DeltaErrors.zOrderingOnPartitionColumnException(colName) - } - if (df.queryExecution.analyzed.resolve(colAttribute.nameParts, isNameEqual).isEmpty) { - throw DeltaErrors.zOrderingColumnDoesNotExistException(colName) + unresolvedZOrderByCols.foreach { colAttribute => + val colName = colAttribute.name + if (checkColStat) { + try { + SchemaUtils.findColumnPosition(colAttribute.nameParts, statCollectionSchema) + } catch { + case e: AnalysisException if e.getMessage.contains("Couldn't find column") => + colsWithoutStats.append(colName) } + } + val isNameEqual = spark.sessionState.conf.resolver + if (partitionColumns.find(isNameEqual(_, colName)).nonEmpty) { + throw DeltaErrors.zOrderingOnPartitionColumnException(colName) + } + if (df.queryExecution.analyzed.resolve(colAttribute.nameParts, isNameEqual).isEmpty) { + throw DeltaErrors.zOrderingColumnDoesNotExistException(colName) + } } if (checkColStat && colsWithoutStats.nonEmpty) { - throw DeltaErrors.zOrderingOnColumnWithNoStatsException(colsWithoutStats.toSeq, spark) + throw DeltaErrors.zOrderingOnColumnWithNoStatsException( + colsWithoutStats.toSeq, spark) } } } @@ -121,15 +122,15 @@ case class OptimizeTableCommand( tableId: Option[TableIdentifier], userPartitionPredicates: Seq[String], options: Map[String, String])(val zOrderBy: Seq[UnresolvedAttribute]) - extends OptimizeTableCommandBase - with LeafRunnableCommand { + extends OptimizeTableCommandBase with LeafRunnableCommand { override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil override def run(sparkSession: SparkSession): Seq[Row] = { + // --- modified start CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession) - val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE", options) + // --- modified end val txn = deltaLog.startTransaction() if (txn.readVersion == -1) { @@ -140,10 +141,12 @@ case class OptimizeTableCommand( // Parse the predicate expression into Catalyst expression and verify only simple filters // on partition columns are present - val partitionPredicates = userPartitionPredicates.flatMap { - predicate => + val partitionPredicates = userPartitionPredicates.flatMap { predicate => val predicates = parsePredicates(sparkSession, predicate) - verifyPartitionPredicates(sparkSession, partitionColumns, predicates) + verifyPartitionPredicates( + sparkSession, + partitionColumns, + predicates) predicates } @@ -155,24 +158,19 @@ case class OptimizeTableCommand( } /** - * Optimize job which compacts small files into larger files to reduce the number of files and - * potentially allow more efficient reads. + * Optimize job which compacts small files into larger files to reduce + * the number of files and potentially allow more efficient reads. * - * @param sparkSession - * Spark environment reference. - * @param txn - * The transaction used to optimize this table - * @param partitionPredicate - * List of partition predicates to select subset of files to optimize. + * @param sparkSession Spark environment reference. + * @param txn The transaction used to optimize this table + * @param partitionPredicate List of partition predicates to select subset of files to optimize. */ class OptimizeExecutor( sparkSession: SparkSession, txn: OptimisticTransaction, partitionPredicate: Seq[Expression], zOrderByColumns: Seq[String]) - extends DeltaCommand - with SQLMetricsReporting - with Serializable { + extends DeltaCommand with SQLMetricsReporting with Serializable { /** Timestamp to use in [[FileAction]] */ private val operationTimestamp = new SystemClock().getTimeMillis() @@ -181,63 +179,72 @@ class OptimizeExecutor( def optimize(): Seq[Row] = { recordDeltaOperation(txn.deltaLog, "delta.optimize") { - val minFileSize = - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE) - val maxFileSize = - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE) + // --- modified start + val isMergeTreeFormat = ClickHouseConfig + .isMergeTreeFormatEngine(txn.deltaLog.unsafeVolatileMetadata.configuration) + // --- modified end + val minFileSize = sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE) + val maxFileSize = sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE) require(minFileSize > 0, "minFileSize must be > 0") require(maxFileSize > 0, "maxFileSize must be > 0") - val candidateFiles = txn.filterFiles(partitionPredicate) + val candidateFiles = txn.filterFiles(partitionPredicate, keepNumRecords = true) val partitionSchema = txn.metadata.partitionSchema - // select all files in case of multi-dimensional clustering - val filesToProcess = candidateFiles.filter(_.size < minFileSize || isMultiDimClustering) - val partitionsToCompact = filesToProcess - .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) - .toSeq - - val jobs = groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) - + val maxDeletedRowsRatio = sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO) + val filesToProcess = pruneCandidateFileList(minFileSize, maxDeletedRowsRatio, candidateFiles) + // --- modified start val maxThreads = sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_THREADS) - val updates = ThreadUtils - .parmap(jobs, "OptimizeJob", maxThreads) { - partitionBinGroup => - runOptimizeBinJobClickhouse( - txn, - partitionBinGroup._1._2, - partitionBinGroup._1._1, - partitionBinGroup._2, - maxFileSize) - } - .flatten + val (updates, jobs) = if (isMergeTreeFormat) { + val partitionsToCompact = filesToProcess + .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) + .toSeq + val jobs = groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) + (ThreadUtils.parmap(jobs, "OptimizeJob", maxThreads) { partitionBinGroup => + // --- modified start + runOptimizeBinJobClickhouse( + txn, + partitionBinGroup._1._2, + partitionBinGroup._1._1, + partitionBinGroup._2, + maxFileSize) + // --- modified end + }.flatten, jobs) + } else { + val partitionsToCompact = filesToProcess.groupBy(_.partitionValues).toSeq + val jobs = groupFilesIntoBins(partitionsToCompact, maxFileSize) + (ThreadUtils.parmap(jobs, "OptimizeJob", maxThreads) { partitionBinGroup => + runOptimizeBinJob(txn, partitionBinGroup._1, partitionBinGroup._2, maxFileSize) + }.flatten, jobs) + } + // --- modified end val addedFiles = updates.collect { case a: AddFile => a } val removedFiles = updates.collect { case r: RemoveFile => r } + val removedDVs = filesToProcess.filter(_.deletionVector != null).map(_.deletionVector).toSeq if (addedFiles.size > 0) { val operation = DeltaOperations.Optimize(partitionPredicate.map(_.sql), zOrderByColumns) - val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles) - commitAndRetry(txn, operation, updates, metrics) { - newTxn => - val newPartitionSchema = newTxn.metadata.partitionSchema - val candidateSetOld = candidateFiles.map(_.path).toSet - val candidateSetNew = newTxn.filterFiles(partitionPredicate).map(_.path).toSet - - // As long as all of the files that we compacted are still part of the table, - // and the partitioning has not changed it is valid to continue to try - // and commit this checkpoint. - if ( - candidateSetOld.subsetOf(candidateSetNew) && partitionSchema == newPartitionSchema - ) { - true - } else { - val deleted = candidateSetOld -- candidateSetNew - logWarning( - s"The following compacted files were delete " + - s"during checkpoint ${deleted.mkString(",")}. Aborting the compaction.") - false - } + val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles, removedDVs) + commitAndRetry(txn, operation, updates, metrics) { newTxn => + val newPartitionSchema = newTxn.metadata.partitionSchema + val candidateSetOld = candidateFiles.map(_.path).toSet + val candidateSetNew = newTxn.filterFiles(partitionPredicate).map(_.path).toSet + + // As long as all of the files that we compacted are still part of the table, + // and the partitioning has not changed it is valid to continue to try + // and commit this checkpoint. + if (candidateSetOld.subsetOf(candidateSetNew) && partitionSchema == newPartitionSchema) { + true + } else { + val deleted = candidateSetOld -- candidateSetNew + logWarning(s"The following compacted files were delete " + + s"during checkpoint ${deleted.mkString(",")}. Aborting the compaction.") + false + } } } @@ -249,37 +256,65 @@ class OptimizeExecutor( optimizeStats.totalConsideredFiles = candidateFiles.size optimizeStats.totalFilesSkipped = optimizeStats.totalConsideredFiles - removedFiles.size optimizeStats.totalClusterParallelism = sparkSession.sparkContext.defaultParallelism + val numTableColumns = txn.snapshot.metadata.schema.size + optimizeStats.numTableColumns = numTableColumns + optimizeStats.numTableColumnsWithStats = + DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(txn.snapshot.metadata) + .min(numTableColumns) + if (removedDVs.size > 0) { + optimizeStats.deletionVectorStats = Some(DeletionVectorStats( + numDeletionVectorsRemoved = removedDVs.size, + numDeletionVectorRowsRemoved = removedDVs.map(_.cardinality).sum)) + } if (isMultiDimClustering) { val inputFileStats = ZOrderFileStats(removedFiles.size, removedFiles.map(_.size.getOrElse(0L)).sum) - optimizeStats.zOrderStats = Some( - ZOrderStats( - strategyName = "all", // means process all files in a partition - inputCubeFiles = ZOrderFileStats(0, 0), - inputOtherFiles = inputFileStats, - inputNumCubes = 0, - mergedFiles = inputFileStats, - // There will one z-cube for each partition - numOutputCubes = optimizeStats.numPartitionsOptimized - )) + optimizeStats.zOrderStats = Some(ZOrderStats( + strategyName = "all", // means process all files in a partition + inputCubeFiles = ZOrderFileStats(0, 0), + inputOtherFiles = inputFileStats, + inputNumCubes = 0, + mergedFiles = inputFileStats, + // There will one z-cube for each partition + numOutputCubes = optimizeStats.numPartitionsOptimized)) } return Seq(Row(txn.deltaLog.dataPath.toString, optimizeStats.toOptimizeMetrics)) } } + /** + * Helper method to prune the list of selected files based on fileSize and ratio of + * deleted rows according to the deletion vector in [[AddFile]]. + */ + private def pruneCandidateFileList( + minFileSize: Long, maxDeletedRowsRatio: Double, files: Seq[AddFile]): Seq[AddFile] = { + + // Select all files in case of multi-dimensional clustering + if (isMultiDimClustering) return files + + def shouldCompactBecauseOfDeletedRows(file: AddFile): Boolean = { + // Always compact files with DVs but without numRecords stats. + // This may be overly aggressive, but it fixes the problem in the long-term, + // as the compacted files will have stats. + (file.deletionVector != null && file.numPhysicalRecords.isEmpty) || + file.deletedToPhysicalRecordsRatio.getOrElse(0d) > maxDeletedRowsRatio + } + + // Select files that are small or have too many deleted rows + files.filter( + addFile => addFile.size < minFileSize || shouldCompactBecauseOfDeletedRows(addFile)) + } + /** * Utility methods to group files into bins for optimize. * - * @param partitionsToCompact - * List of files to compact group by partition. Partition is defined by the partition values - * (partCol -> partValue) - * @param maxTargetFileSize - * Max size (in bytes) of the compaction output file. - * @return - * Sequence of bins. Each bin contains one or more files from the same partition and targeted - * for one output file. + * @param partitionsToCompact List of files to compact group by partition. + * Partition is defined by the partition values (partCol -> partValue) + * @param maxTargetFileSize Max size (in bytes) of the compaction output file. + * @return Sequence of bins. Each bin contains one or more files from the same + * partition and targeted for one output file. */ private def groupFilesIntoBins( partitionsToCompact: Seq[(Map[String, String], Seq[AddFile])], @@ -291,47 +326,42 @@ class OptimizeExecutor( val currentBin = new ArrayBuffer[AddFile]() var currentBinSize = 0L - files.sortBy(_.size).foreach { - file => - // Generally, a bin is a group of existing files, whose total size does not exceed the - // desired maxFileSize. They will be coalesced into a single output file. - // However, if isMultiDimClustering = true, all files in a partition will be read by the - // same job, the data will be range-partitioned and - // umFiles = totalFileSize / maxFileSize will be produced. See below. - if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) { - bins += currentBin.toVector - currentBin.clear() - currentBin += file - currentBinSize = file.size - } else { - currentBin += file - currentBinSize += file.size - } + files.sortBy(_.size).foreach { file => + // Generally, a bin is a group of existing files, whose total size does not exceed the + // desired maxFileSize. They will be coalesced into a single output file. + // However, if isMultiDimClustering = true, all files in a partition will be read by the + // same job, the data will be range-partitioned and numFiles = totalFileSize / maxFileSize + // will be produced. See below. + if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) { + bins += currentBin.toVector + currentBin.clear() + currentBin += file + currentBinSize = file.size + } else { + currentBin += file + currentBinSize += file.size + } } if (currentBin.nonEmpty) { bins += currentBin.toVector } - bins - .map(b => (partition, b)) - // select bins that have at least two files or in case of multi-dim clustering - // select all bins - .filter(_._2.size > 1 || isMultiDimClustering) + bins.filter { bin => + bin.size > 1 || // bin has more than one file or + (bin.size == 1 && bin(0).deletionVector != null) || // single file in the bin has a DV or + isMultiDimClustering // multi-clustering + }.map(b => (partition, b)) } } /** * Utility method to run a Spark job to compact the files in given bin * - * @param txn - * [[OptimisticTransaction]] instance in use to commit the changes to DeltaLog. - * @param partition - * Partition values of the partition that files in [[bin]] belongs to. - * @param bin - * List of files to compact into one large file. - * @param maxFileSize - * Targeted output file size in bytes + * @param txn [[OptimisticTransaction]] instance in use to commit the changes to DeltaLog. + * @param partition Partition values of the partition that files in [[bin]] belongs to. + * @param bin List of files to compact into one large file. + * @param maxFileSize Targeted output file size in bytes */ private def runOptimizeBinJob( txn: OptimisticTransaction, @@ -344,10 +374,13 @@ class OptimizeExecutor( val repartitionDF = if (isMultiDimClustering) { val totalSize = bin.map(_.size).sum val approxNumFiles = Math.max(1, totalSize / maxFileSize).toInt - MultiDimClustering.cluster(input, approxNumFiles, zOrderByColumns) + MultiDimClustering.cluster( + input, + approxNumFiles, + zOrderByColumns) } else { - val useRepartition = - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_REPARTITION_ENABLED) + val useRepartition = sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_REPARTITION_ENABLED) if (useRepartition) { input.repartition(numPartitions = 1) } else { @@ -369,7 +402,7 @@ class OptimizeExecutor( case other => throw new IllegalStateException( s"Unexpected action $other with type ${other.getClass}. File compaction job output" + - s"should only have AddFiles") + s"should only have AddFiles") } val removeFiles = bin.map(f => f.removeWithTimestamp(operationTimestamp, dataChange = false)) val updates = addFiles ++ removeFiles @@ -377,9 +410,9 @@ class OptimizeExecutor( } /** - * Attempts to commit the given actions to the log. In the case of a concurrent update, the given - * function will be invoked with a new transaction to allow custom conflict detection logic to - * indicate it is safe to try again, by returning `true`. + * Attempts to commit the given actions to the log. In the case of a concurrent update, + * the given function will be invoked with a new transaction to allow custom conflict + * detection logic to indicate it is safe to try again, by returning `true`. * * This function will continue to try to commit to the log as long as `f` returns `true`, * otherwise throws a subclass of [[ConcurrentModificationException]]. @@ -409,7 +442,8 @@ class OptimizeExecutor( private def createMetrics( sparkContext: SparkContext, addedFiles: Seq[AddFile], - removedFiles: Seq[RemoveFile]): Map[String, SQLMetric] = { + removedFiles: Seq[RemoveFile], + removedDVs: Seq[DeletionVectorDescriptor]): Map[String, SQLMetric] = { def setAndReturnMetric(description: String, value: Long) = { val metric = createMetric(sparkContext, description) @@ -419,19 +453,37 @@ class OptimizeExecutor( def totalSize(actions: Seq[FileAction]): Long = { var totalSize = 0L - actions.foreach { - file => - val fileSize = file match { - case addFile: AddFile => addFile.size - case removeFile: RemoveFile => removeFile.size.getOrElse(0L) - case default => - throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") - } - totalSize += fileSize + actions.foreach { file => + val fileSize = file match { + case addFile: AddFile => addFile.size + case removeFile: RemoveFile => removeFile.size.getOrElse(0L) + case default => + throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") + } + totalSize += fileSize } totalSize } + val (deletionVectorRowsRemoved, deletionVectorBytesRemoved) = + removedDVs.map(dv => (dv.cardinality, dv.sizeInBytes.toLong)) + .reduceLeftOption((dv1, dv2) => (dv1._1 + dv2._1, dv1._2 + dv2._2)) + .getOrElse((0L, 0L)) + + val dvMetrics: Map[String, SQLMetric] = Map( + "numDeletionVectorsRemoved" -> + setAndReturnMetric( + "total number of deletion vectors removed", + removedDVs.size), + "numDeletionVectorRowsRemoved" -> + setAndReturnMetric( + "total number of deletion vector rows removed", + deletionVectorRowsRemoved), + "numDeletionVectorBytesRemoved" -> + setAndReturnMetric( + "total number of bytes of removed deletion vectors", + deletionVectorBytesRemoved)) + val sizeStats = FileSizeStatsWithHistogram.create(addedFiles.map(_.size).sorted) Map[String, SQLMetric]( "minFileSize" -> setAndReturnMetric("minimum file size", sizeStats.get.min), @@ -444,6 +496,6 @@ class OptimizeExecutor( "numAddedBytes" -> setAndReturnMetric("total number of bytes added", totalSize(addedFiles)), "numRemovedBytes" -> setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)) - ) + ) ++ dvMetrics } } diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 66% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala index ad118470fc7f..94ccef961c08 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/commands/UpdateCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala @@ -14,32 +14,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta.commands +// scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaLog, DeltaOperations, DeltaTableUtils, DeltaUDF, OptimisticTransaction} +import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} +import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC, CDC_TYPE_UPDATE_POSTIMAGE, CDC_TYPE_UPDATE_PREIMAGE} +import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} +import org.apache.hadoop.fs.Path + import org.apache.spark.SparkContext -import org.apache.spark.sql._ +import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Expression, If, Literal} import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC, CDC_TYPE_UPDATE_POSTIMAGE, CDC_TYPE_UPDATE_PREIMAGE} -import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} -import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.LongType -// scalastyle:off import.ordering.noEmptyLine -import org.apache.hadoop.fs.Path - /** * Gluten overwrite Delta: * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: + * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: * 1. In Clickhouse backend, we can't implement input_file_name() correctly, we can only implement * it so that it return a a list of filenames (concated by ','). */ @@ -47,18 +47,18 @@ import org.apache.hadoop.fs.Path /** * Performs an Update using `updateExpression` on the rows that match `condition` * - * Algorithm: 1) Identify the affected files, i.e., the files that may have the rows to be updated. - * 2) Scan affected files, apply the updates, and generate a new DF with updated rows. 3) Use the - * Delta protocol to atomically write the new DF as new files and remove the affected files that are - * identified in step 1. + * Algorithm: + * 1) Identify the affected files, i.e., the files that may have the rows to be updated. + * 2) Scan affected files, apply the updates, and generate a new DF with updated rows. + * 3) Use the Delta protocol to atomically write the new DF as new files and remove + * the affected files that are identified in step 1. */ case class UpdateCommand( tahoeFileIndex: TahoeFileIndex, target: LogicalPlan, updateExpressions: Seq[Expression], condition: Option[Expression]) - extends LeafRunnableCommand - with DeltaCommand { + extends LeafRunnableCommand with DeltaCommand { override val output: Seq[Attribute] = { Seq(AttributeReference("num_affected_rows", LongType)()) @@ -70,7 +70,9 @@ case class UpdateCommand( override lazy val metrics = Map[String, SQLMetric]( "numAddedFiles" -> createMetric(sc, "number of files added."), + "numAddedBytes" -> createMetric(sc, "number of bytes added"), "numRemovedFiles" -> createMetric(sc, "number of files removed."), + "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), "numUpdatedRows" -> createMetric(sc, "number of rows updated."), "numCopiedRows" -> createMetric(sc, "number of rows copied."), "executionTimeMs" -> @@ -87,8 +89,14 @@ case class UpdateCommand( final override def run(sparkSession: SparkSession): Seq[Row] = { recordDeltaOperation(tahoeFileIndex.deltaLog, "delta.dml.update") { val deltaLog = tahoeFileIndex.deltaLog - deltaLog.assertRemovable() - deltaLog.withNewTransaction(txn => performUpdate(sparkSession, deltaLog, txn)) + deltaLog.withNewTransaction { txn => + DeltaLog.assertRemovable(txn.snapshot) + if (hasBeenExecuted(txn, sparkSession)) { + sendDriverMetrics(sparkSession, metrics) + return Seq.empty + } + performUpdate(sparkSession, deltaLog, txn) + } // Re-cache all cached plans(including this relation itself, if it's cached) that refer to // this data source relation. sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) @@ -97,13 +105,13 @@ case class UpdateCommand( } private def performUpdate( - sparkSession: SparkSession, - deltaLog: DeltaLog, - txn: OptimisticTransaction): Unit = { + sparkSession: SparkSession, deltaLog: DeltaLog, txn: OptimisticTransaction): Unit = { import org.apache.spark.sql.delta.implicits._ var numTouchedFiles: Long = 0 var numRewrittenFiles: Long = 0 + var numAddedBytes: Long = 0 + var numRemovedBytes: Long = 0 var numAddedChangeFiles: Long = 0 var changeFileBytes: Long = 0 var scanTimeMs: Long = 0 @@ -115,9 +123,7 @@ case class UpdateCommand( val updateCondition = condition.getOrElse(Literal.TrueLiteral) val (metadataPredicates, dataPredicates) = DeltaTableUtils.splitMetadataAndDataPredicates( - updateCondition, - txn.metadata.partitionColumns, - sparkSession) + updateCondition, txn.metadata.partitionColumns, sparkSession) val candidateFiles = txn.filterFiles(metadataPredicates ++ dataPredicates) val nameToAddFile = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) @@ -134,34 +140,27 @@ case class UpdateCommand( } else { // Case 3: Find all the affected files using the user-specified condition val fileIndex = new TahoeBatchFileIndex( - sparkSession, - "update", - candidateFiles, - deltaLog, - tahoeFileIndex.path, - txn.snapshot) + sparkSession, "update", candidateFiles, deltaLog, tahoeFileIndex.path, txn.snapshot) // Keep everything from the resolved target except a new TahoeFileIndex // that only involves the affected files instead of all files. val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) val data = Dataset.ofRows(sparkSession, newTarget) val updatedRowCount = metrics("numUpdatedRows") - val updatedRowUdf = DeltaUDF - .boolean { - () => - updatedRowCount += 1 - true - } - .asNondeterministic() + val updatedRowUdf = DeltaUDF.boolean { () => + updatedRowCount += 1 + true + }.asNondeterministic() val pathsToRewrite = withStatusCode("DELTA", UpdateCommand.FINDING_TOUCHED_FILES_MSG) { - data - .filter(new Column(updateCondition)) + // --- modified start + data.filter(new Column(updateCondition)) .select(input_file_name().as("input_files")) .filter(updatedRowUdf()) .select(explode(split(col("input_files"), ","))) .distinct() .as[String] .collect() + // --- modified end } scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 @@ -177,13 +176,8 @@ case class UpdateCommand( } else { // Generate the new files containing the updated values withStatusCode("DELTA", UpdateCommand.rewritingFilesMsg(filesToRewrite.size)) { - rewriteFiles( - sparkSession, - txn, - tahoeFileIndex.path, - filesToRewrite.map(_.path), - nameToAddFile, - updateCondition) + rewriteFiles(sparkSession, txn, tahoeFileIndex.path, + filesToRewrite.map(_.path), nameToAddFile, updateCondition) } } @@ -191,6 +185,7 @@ case class UpdateCommand( val (changeActions, addActions) = newActions.partition(_.isInstanceOf[AddCDCFile]) numRewrittenFiles = addActions.size + numAddedBytes = addActions.map(_.getFileSize).sum numAddedChangeFiles = changeActions.size changeFileBytes = changeActions.collect { case f: AddCDCFile => f.size }.sum @@ -202,47 +197,42 @@ case class UpdateCommand( // files containing the updated values val operationTimestamp = System.currentTimeMillis() val deleteActions = filesToRewrite.map(_.removeWithTimestamp(operationTimestamp)) - + numRemovedBytes = filesToRewrite.map(_.getFileSize).sum deleteActions ++ newActions } - if (totalActions.nonEmpty) { - metrics("numAddedFiles").set(numRewrittenFiles) - metrics("numAddedChangeFiles").set(numAddedChangeFiles) - metrics("changeFileBytes").set(changeFileBytes) - metrics("numRemovedFiles").set(numTouchedFiles) - metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) - metrics("scanTimeMs").set(scanTimeMs) - metrics("rewriteTimeMs").set(rewriteTimeMs) - // In the case where the numUpdatedRows is not captured, we can siphon out the metrics from - // the BasicWriteStatsTracker. This is for case 2 where the update condition contains only - // metadata predicates and so the entire partition is re-written. - val outputRows = txn.getMetric("numOutputRows").map(_.value).getOrElse(-1L) - if ( - metrics("numUpdatedRows").value == 0 && outputRows != 0 && - metrics("numCopiedRows").value == 0 - ) { - // We know that numTouchedRows = numCopiedRows + numUpdatedRows. - // Since an entire partition was re-written, no rows were copied. - // So numTouchedRows == numUpdateRows - metrics("numUpdatedRows").set(metrics("numTouchedRows").value) - } else { - // This is for case 3 where the update condition contains both metadata and data predicates - // so relevant files will have some rows updated and some rows copied. We don't need to - // consider case 1 here, where no files match the update condition, as we know that - // `totalActions` is empty. - metrics("numCopiedRows").set( - metrics("numTouchedRows").value - metrics("numUpdatedRows").value) - } - txn.registerSQLMetrics(sparkSession, metrics) - txn.commit(totalActions, DeltaOperations.Update(condition.map(_.toString))) - // This is needed to make the SQL metrics visible in the Spark UI - val executionId = sparkSession.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates( - sparkSession.sparkContext, - executionId, - metrics.values.toSeq) + metrics("numAddedFiles").set(numRewrittenFiles) + metrics("numAddedBytes").set(numAddedBytes) + metrics("numAddedChangeFiles").set(numAddedChangeFiles) + metrics("changeFileBytes").set(changeFileBytes) + metrics("numRemovedFiles").set(numTouchedFiles) + metrics("numRemovedBytes").set(numRemovedBytes) + metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) + metrics("scanTimeMs").set(scanTimeMs) + metrics("rewriteTimeMs").set(rewriteTimeMs) + // In the case where the numUpdatedRows is not captured, we can siphon out the metrics from + // the BasicWriteStatsTracker. This is for case 2 where the update condition contains only + // metadata predicates and so the entire partition is re-written. + val outputRows = txn.getMetric("numOutputRows").map(_.value).getOrElse(-1L) + if (metrics("numUpdatedRows").value == 0 && outputRows != 0 && + metrics("numCopiedRows").value == 0) { + // We know that numTouchedRows = numCopiedRows + numUpdatedRows. + // Since an entire partition was re-written, no rows were copied. + // So numTouchedRows == numUpdateRows + metrics("numUpdatedRows").set(metrics("numTouchedRows").value) + } else { + // This is for case 3 where the update condition contains both metadata and data predicates + // so relevant files will have some rows updated and some rows copied. We don't need to + // consider case 1 here, where no files match the update condition, as we know that + // `totalActions` is empty. + metrics("numCopiedRows").set( + metrics("numTouchedRows").value - metrics("numUpdatedRows").value) } + txn.registerSQLMetrics(sparkSession, metrics) + + val finalActions = createSetTransaction(sparkSession, deltaLog).toSeq ++ totalActions + txn.commitIfNeeded(finalActions, DeltaOperations.Update(condition.map(_.toString))) + sendDriverMetrics(sparkSession, metrics) recordDeltaEvent( deltaLog, @@ -255,19 +245,17 @@ case class UpdateCommand( numAddedChangeFiles, changeFileBytes, scanTimeMs, - rewriteTimeMs - ) + rewriteTimeMs) ) } /** * Scan all the affected files and write out the updated files. * - * When CDF is enabled, includes the generation of CDC preimage and postimage columns for changed - * rows. + * When CDF is enabled, includes the generation of CDC preimage and postimage columns for + * changed rows. * - * @return - * the list of [[AddFile]]s and [[AddCDCFile]]s that have been written. + * @return the list of [[AddFile]]s and [[AddCDCFile]]s that have been written. */ private def rewriteFiles( spark: SparkSession, @@ -277,21 +265,18 @@ case class UpdateCommand( nameToAddFileMap: Map[String, AddFile], condition: Expression): Seq[FileAction] = { // Containing the map from the relative file path to AddFile - val baseRelation = - buildBaseRelation(spark, txn, "update", rootPath, inputLeafFiles, nameToAddFileMap) + val baseRelation = buildBaseRelation( + spark, txn, "update", rootPath, inputLeafFiles, nameToAddFileMap) val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) val targetDf = Dataset.ofRows(spark, newTarget) // Number of total rows that we have seen, i.e. are either copying or updating (sum of both). // This will be used later, along with numUpdatedRows, to determine numCopiedRows. val numTouchedRows = metrics("numTouchedRows") - val numTouchedRowsUdf = DeltaUDF - .boolean { - () => - numTouchedRows += 1 - true - } - .asNondeterministic() + val numTouchedRowsUdf = DeltaUDF.boolean { () => + numTouchedRows += 1 + true + }.asNondeterministic() val updatedDataFrame = UpdateCommand.withUpdatedColumns( target, @@ -300,8 +285,7 @@ case class UpdateCommand( targetDf .filter(numTouchedRowsUdf()) .withColumn(UpdateCommand.CONDITION_COLUMN_NAME, new Column(condition)), - UpdateCommand.shouldOutputCdc(txn) - ) + UpdateCommand.shouldOutputCdc(txn)) txn.writeFiles(updatedDataFrame) } @@ -324,25 +308,20 @@ object UpdateCommand { } /** - * Build the new columns. If the condition matches, generate the new value using the corresponding - * UPDATE EXPRESSION; otherwise, keep the original column value. + * Build the new columns. If the condition matches, generate the new value using + * the corresponding UPDATE EXPRESSION; otherwise, keep the original column value. * * When CDC is enabled, includes the generation of CDC pre-image and post-image columns for * changed rows. * - * @param target - * target we are updating into - * @param updateExpressions - * the update transformation to perform on the input DataFrame - * @param dfWithEvaluatedCondition - * source DataFrame on which we will apply the update expressions with an additional column - * CONDITION_COLUMN_NAME which is the true/false value of if the update condition is satisfied - * @param condition - * update condition - * @param shouldOutputCdc - * if we should output CDC data during this UPDATE operation. - * @return - * the updated DataFrame, with extra CDC columns if CDC is enabled + * @param target target we are updating into + * @param updateExpressions the update transformation to perform on the input DataFrame + * @param dfWithEvaluatedCondition source DataFrame on which we will apply the update expressions + * with an additional column CONDITION_COLUMN_NAME which is the + * true/false value of if the update condition is satisfied + * @param condition update condition + * @param shouldOutputCdc if we should output CDC data during this UPDATE operation. + * @return the updated DataFrame, with extra CDC columns if CDC is enabled */ def withUpdatedColumns( target: LogicalPlan, @@ -377,24 +356,22 @@ object UpdateCommand { If( UnresolvedAttribute(CONDITION_COLUMN_NAME), packedUpdates, // if it should be updated, then use `packagedUpdates` - array(struct(noopRewriteCols: _*)).expr - ) // else, this is a noop rewrite + array(struct(noopRewriteCols: _*)).expr) // else, this is a noop rewrite } // Explode the packed array, and project back out the final data columns. val finalColNames = target.output.map(_.name) :+ CDC_TYPE_COLUMN_NAME dfWithEvaluatedCondition .select(explode(new Column(packedData)).as("packedData")) - .select(finalColNames.map(n => col(s"packedData.`$n`").as(s"$n")): _*) + .select(finalColNames.map { n => col(s"packedData.`$n`").as(s"$n") }: _*) } else { - val finalCols = updateExpressions.zip(target.output).map { - case (update, original) => - val updated = if (condition == Literal.TrueLiteral) { - update - } else { - If(UnresolvedAttribute(CONDITION_COLUMN_NAME), update, original) - } - new Column(Alias(updated, original.name)()) + val finalCols = updateExpressions.zip(target.output).map { case (update, original) => + val updated = if (condition == Literal.TrueLiteral) { + update + } else { + If(UnresolvedAttribute(CONDITION_COLUMN_NAME), update, original) + } + new Column(Alias(updated, original.name)()) } dfWithEvaluatedCondition.select(finalCols: _*) @@ -407,25 +384,16 @@ object UpdateCommand { /** * Used to report details about update. * - * @param condition: - * what was the update condition - * @param numFilesTotal: - * how big is the table - * @param numTouchedFiles: - * how many files did we touch - * @param numRewrittenFiles: - * how many files had to be rewritten - * @param numAddedChangeFiles: - * how many change files were generated - * @param changeFileBytes: - * total size of change files generated - * @param scanTimeMs: - * how long did finding take - * @param rewriteTimeMs: - * how long did rewriting take + * @param condition: what was the update condition + * @param numFilesTotal: how big is the table + * @param numTouchedFiles: how many files did we touch + * @param numRewrittenFiles: how many files had to be rewritten + * @param numAddedChangeFiles: how many change files were generated + * @param changeFileBytes: total size of change files generated + * @param scanTimeMs: how long did finding take + * @param rewriteTimeMs: how long did rewriting take * - * @note - * All the time units are milliseconds. + * @note All the time units are milliseconds. */ case class UpdateMetric( condition: String, diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala new file mode 100644 index 000000000000..5be548caf01c --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -0,0 +1,575 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands + +// scalastyle:off import.ordering.noEmptyLine +import java.net.URI +import java.util.Date +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.DeltaFileOperations +import org.apache.spark.sql.delta.util.DeltaFileOperations.tryDeleteNonRecursive +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, Path} + +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric +import org.apache.spark.sql.functions._ +import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 2.3.0. It is modified to overcome the following issues: + * 1. In Gluten, part is a directory, but VacuumCommand assumes part is a file. So we need some + * modifications to make it work. + */ + +/** + * Vacuums the table by clearing all untracked files and folders within this table. + * First lists all the files and directories in the table, and gets the relative paths with + * respect to the base of the table. Then it gets the list of all tracked files for this table, + * which may or may not be within the table base path, and gets the relative paths of + * all the tracked files with respect to the base of the table. Files outside of the table path + * will be ignored. Then we take a diff of the files and delete directories that were already empty, + * and all files that are within the table that are no longer tracked. + */ +object VacuumCommand extends VacuumCommandImpl with Serializable { + + // --- modified start + case class FileNameAndSize(path: String, length: Long, isDir: Boolean = false) + // --- modified end + /** + * Additional check on retention duration to prevent people from shooting themselves in the foot. + */ + protected def checkRetentionPeriodSafety( + spark: SparkSession, + retentionMs: Option[Long], + configuredRetention: Long): Unit = { + require(retentionMs.forall(_ >= 0), "Retention for Vacuum can't be less than 0.") + val checkEnabled = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) + val retentionSafe = retentionMs.forall(_ >= configuredRetention) + var configuredRetentionHours = TimeUnit.MILLISECONDS.toHours(configuredRetention) + if (TimeUnit.HOURS.toMillis(configuredRetentionHours) < configuredRetention) { + configuredRetentionHours += 1 + } + require(!checkEnabled || retentionSafe, + s"""Are you sure you would like to vacuum files with such a low retention period? If you have + |writers that are currently writing to this table, there is a risk that you may corrupt the + |state of your Delta table. + | + |If you are certain that there are no operations being performed on this table, such as + |insert/upsert/delete/optimize, then you may turn off this check by setting: + |spark.databricks.delta.retentionDurationCheck.enabled = false + | + |If you are not sure, please use a value not less than "$configuredRetentionHours hours". + """.stripMargin) + } + + /** + * Clears all untracked files and folders within this table. First lists all the files and + * directories in the table, and gets the relative paths with respect to the base of the + * table. Then it gets the list of all tracked files for this table, which may or may not + * be within the table base path, and gets the relative paths of all the tracked files with + * respect to the base of the table. Files outside of the table path will be ignored. + * Then we take a diff of the files and delete directories that were already empty, and all files + * that are within the table that are no longer tracked. + * + * @param dryRun If set to true, no files will be deleted. Instead, we will list all files and + * directories that will be cleared. + * @param retentionHours An optional parameter to override the default Delta tombstone retention + * period + * @return A Dataset containing the paths of the files/folders to delete in dryRun mode. Otherwise + * returns the base path of the table. + */ + def gc( + spark: SparkSession, + deltaLog: DeltaLog, + dryRun: Boolean = true, + retentionHours: Option[Double] = None, + clock: Clock = new SystemClock): DataFrame = { + recordDeltaOperation(deltaLog, "delta.gc") { + + val path = deltaLog.dataPath + val deltaHadoopConf = deltaLog.newDeltaHadoopConf() + val fs = path.getFileSystem(deltaHadoopConf) + + import org.apache.spark.sql.delta.implicits._ + + val snapshot = deltaLog.update() + + require(snapshot.version >= 0, "No state defined for this table. Is this really " + + "a Delta table? Refusing to garbage collect.") + + // --- modified start + val isMergeTreeFormat = ClickHouseConfig + .isMergeTreeFormatEngine(deltaLog.unsafeVolatileMetadata.configuration) + // --- modified end + + DeletionVectorUtils.assertDeletionVectorsNotReadable( + spark, snapshot.metadata, snapshot.protocol) + + val snapshotTombstoneRetentionMillis = DeltaLog.tombstoneRetentionMillis(snapshot.metadata) + val retentionMillis = retentionHours.map(h => TimeUnit.HOURS.toMillis(math.round(h))) + checkRetentionPeriodSafety(spark, retentionMillis, snapshotTombstoneRetentionMillis) + + val deleteBeforeTimestamp = retentionMillis.map { millis => + clock.getTimeMillis() - millis + }.getOrElse(snapshot.minFileRetentionTimestamp) + // --- modified start: toGMTString is a deprecated function + logInfo(s"Starting garbage collection (dryRun = $dryRun) of untracked files older than " + + s"${new Date(deleteBeforeTimestamp).toString} in $path") + // --- modified end + val hadoopConf = spark.sparkContext.broadcast( + new SerializableConfiguration(deltaHadoopConf)) + val basePath = fs.makeQualified(path).toString + var isBloomFiltered = false + val parallelDeleteEnabled = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_ENABLED) + val parallelDeletePartitions = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM) + .getOrElse(spark.sessionState.conf.numShufflePartitions) + val relativizeIgnoreError = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) + val startTimeToIdentifyEligibleFiles = System.currentTimeMillis() + val validFiles = snapshot.stateDS + .mapPartitions { actions => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + actions.flatMap { + _.unwrap match { + case tombstone: RemoveFile if tombstone.delTimestamp < deleteBeforeTimestamp => + Nil + case fa: FileAction => + getValidRelativePathsAndSubdirs( + fa, + fs, + reservoirBase, + relativizeIgnoreError, + isBloomFiltered) + case _ => Nil + } + } + }.toDF("path") + + val partitionColumns = snapshot.metadata.partitionSchema.fieldNames + val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism + + val allFilesAndDirs = DeltaFileOperations.recursiveListDirs( + spark, + Seq(basePath), + hadoopConf, + hiddenDirNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), + hiddenFileNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), + fileListingParallelism = Option(parallelism) + ) + .groupByKey(_.path) + .mapGroups { (k, v) => + val duplicates = v.toSeq + // of all the duplicates we can return the newest file. + duplicates.maxBy(_.modificationTime) + } + + try { + allFilesAndDirs.cache() + + implicit val fileNameAndSizeEncoder = org.apache.spark.sql.Encoders.product[FileNameAndSize] + + val dirCounts = allFilesAndDirs.where(col("isDir")).count() + 1 // +1 for the base path + + // The logic below is as follows: + // 1. We take all the files and directories listed in our reservoir + // 2. We filter all files older than our tombstone retention period and directories + // 3. We get the subdirectories of all files so that we can find non-empty directories + // 4. We groupBy each path, and count to get how many files are in each sub-directory + // 5. We subtract all the valid files and tombstones in our state + // 6. We filter all paths with a count of 1, which will correspond to files not in the + // state, and empty directories. We can safely delete all of these + // --- modified start + val diff = if (isMergeTreeFormat) { + val diff_tmp = allFilesAndDirs + .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) + .mapPartitions { fileStatusIterator => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + fileStatusIterator.flatMap { fileStatus => + if (fileStatus.isDir) { + Iterator.single(FileNameAndSize( + relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), + 0L, + true)) + } else { + val dirs = getAllSubdirs(basePath, fileStatus.path, fs) + val dirsWithSlash = dirs.map { p => + val relativizedPath = relativize(new Path(p), fs, reservoirBase, isDir = true) + FileNameAndSize(relativizedPath, 0L, true) + } + dirsWithSlash ++ Iterator( + FileNameAndSize(relativize( + fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), + fileStatus.length)) + } + } + } + .withColumn( + "dir", + when(col("isDir"), col("path")) + .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) + .groupBy(col("path"), col("dir")) + .agg(count(new Column("*")).as("count"), sum("length").as("length")) + + diff_tmp + .join(validFiles, diff_tmp("dir") === validFiles("path"), "leftanti") + .where(col("count") === 1) + } else { + allFilesAndDirs + .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) + .mapPartitions { fileStatusIterator => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + fileStatusIterator.flatMap { fileStatus => + if (fileStatus.isDir) { + Iterator.single(FileNameAndSize( + relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), 0L)) + } else { + val dirs = getAllSubdirs(basePath, fileStatus.path, fs) + val dirsWithSlash = dirs.map { p => + val relativizedPath = relativize(new Path(p), fs, reservoirBase, isDir = true) + FileNameAndSize(relativizedPath, 0L) + } + dirsWithSlash ++ Iterator( + FileNameAndSize(relativize( + fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), + fileStatus.length)) + } + } + } + .groupBy(col("path")) + .agg(count(new Column("*")).as("count"), sum("length").as("length")) + .join(validFiles, Seq("path"), "leftanti") + .where(col("count") === 1) + } + // --- modified end + + val sizeOfDataToDeleteRow = diff.agg(sum("length").cast("long")).first + val sizeOfDataToDelete = if (sizeOfDataToDeleteRow.isNullAt(0)) { + 0L + } else { + sizeOfDataToDeleteRow.getLong(0) + } + + val diffFiles = diff + .select(col("path")) + .as[String] + .map { relativePath => + assert(!stringToPath(relativePath).isAbsolute, + "Shouldn't have any absolute paths for deletion here.") + pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) + } + val timeTakenToIdentifyEligibleFiles = + System.currentTimeMillis() - startTimeToIdentifyEligibleFiles + + val numFiles = diffFiles.count() + if (dryRun) { + val stats = DeltaVacuumStats( + isDryRun = true, + specifiedRetentionMillis = retentionMillis, + defaultRetentionMillis = snapshotTombstoneRetentionMillis, + minRetainedTimestamp = deleteBeforeTimestamp, + dirsPresentBeforeDelete = dirCounts, + objectsDeleted = numFiles, + sizeOfDataToDelete = sizeOfDataToDelete, + timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, + timeTakenForDelete = 0L) + + recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) + logConsole(s"Found $numFiles files ($sizeOfDataToDelete bytes) and directories in " + + s"a total of $dirCounts directories that are safe to delete.") + + return diffFiles.map(f => stringToPath(f).toString).toDF("path") + } + logVacuumStart( + spark, + deltaLog, + path, + diffFiles, + sizeOfDataToDelete, + retentionMillis, + snapshotTombstoneRetentionMillis) + + val deleteStartTime = System.currentTimeMillis() + val filesDeleted = try { + delete(diffFiles, spark, basePath, + hadoopConf, parallelDeleteEnabled, parallelDeletePartitions) + } catch { + case t: Throwable => + logVacuumEnd(deltaLog, spark, path) + throw t + } + val timeTakenForDelete = System.currentTimeMillis() - deleteStartTime + val stats = DeltaVacuumStats( + isDryRun = false, + specifiedRetentionMillis = retentionMillis, + defaultRetentionMillis = snapshotTombstoneRetentionMillis, + minRetainedTimestamp = deleteBeforeTimestamp, + dirsPresentBeforeDelete = dirCounts, + objectsDeleted = filesDeleted, + sizeOfDataToDelete = sizeOfDataToDelete, + timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, + timeTakenForDelete = timeTakenForDelete) + recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) + logVacuumEnd(deltaLog, spark, path, Some(filesDeleted), Some(dirCounts)) + + + spark.createDataset(Seq(basePath)).toDF("path") + } finally { + allFilesAndDirs.unpersist() + } + } + } +} + +trait VacuumCommandImpl extends DeltaCommand { + + private val supportedFsForLogging = Seq( + "wasbs", "wasbss", "abfs", "abfss", "adl", "gs", "file", "hdfs" + ) + + /** + * Returns whether we should record vacuum metrics in the delta log. + */ + private def shouldLogVacuum( + spark: SparkSession, + deltaLog: DeltaLog, + hadoopConf: Configuration, + path: Path): Boolean = { + val logVacuumConf = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_LOGGING_ENABLED) + + if (logVacuumConf.nonEmpty) { + return logVacuumConf.get + } + + val logStore = deltaLog.store + + try { + val rawResolvedUri: URI = logStore.resolvePathOnPhysicalStorage(path, hadoopConf).toUri + val scheme = rawResolvedUri.getScheme + supportedFsForLogging.contains(scheme) + } catch { + case _: UnsupportedOperationException => + logWarning("Vacuum event logging" + + " not enabled on this file system because we cannot detect your cloud storage type.") + false + } + } + + /** + * Record Vacuum specific metrics in the commit log at the START of vacuum. + * + * @param spark - spark session + * @param deltaLog - DeltaLog of the table + * @param path - the (data) path to the root of the table + * @param diff - the list of paths (files, directories) that are safe to delete + * @param sizeOfDataToDelete - the amount of data (bytes) to be deleted + * @param specifiedRetentionMillis - the optional override retention period (millis) to keep + * logically removed files before deleting them + * @param defaultRetentionMillis - the default retention period (millis) + */ + protected def logVacuumStart( + spark: SparkSession, + deltaLog: DeltaLog, + path: Path, + diff: Dataset[String], + sizeOfDataToDelete: Long, + specifiedRetentionMillis: Option[Long], + defaultRetentionMillis: Long): Unit = { + logInfo(s"Deleting untracked files and empty directories in $path. The amount of data to be " + + s"deleted is $sizeOfDataToDelete (in bytes)") + + // We perform an empty commit in order to record information about the Vacuum + if (shouldLogVacuum(spark, deltaLog, deltaLog.newDeltaHadoopConf(), path)) { + val checkEnabled = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) + val txn = deltaLog.startTransaction() + val metrics = Map[String, SQLMetric]( + "numFilesToDelete" -> createMetric(spark.sparkContext, "number of files to deleted"), + "sizeOfDataToDelete" -> createMetric(spark.sparkContext, + "The total amount of data to be deleted in bytes") + ) + metrics("numFilesToDelete").set(diff.count()) + metrics("sizeOfDataToDelete").set(sizeOfDataToDelete) + txn.registerSQLMetrics(spark, metrics) + txn.commit(actions = Seq(), DeltaOperations.VacuumStart( + checkEnabled, + specifiedRetentionMillis, + defaultRetentionMillis + )) + } + } + + /** + * Record Vacuum specific metrics in the commit log at the END of vacuum. + * + * @param deltaLog - DeltaLog of the table + * @param spark - spark session + * @param path - the (data) path to the root of the table + * @param filesDeleted - if the vacuum completed this will contain the number of files deleted. + * if the vacuum failed, this will be None. + * @param dirCounts - if the vacuum completed this will contain the number of directories + * vacuumed. if the vacuum failed, this will be None. + */ + protected def logVacuumEnd( + deltaLog: DeltaLog, + spark: SparkSession, + path: Path, + filesDeleted: Option[Long] = None, + dirCounts: Option[Long] = None): Unit = { + if (shouldLogVacuum(spark, deltaLog, deltaLog.newDeltaHadoopConf(), path)) { + val txn = deltaLog.startTransaction() + val status = if (filesDeleted.isEmpty && dirCounts.isEmpty) { "FAILED" } else { "COMPLETED" } + if (filesDeleted.nonEmpty && dirCounts.nonEmpty) { + val metrics = Map[String, SQLMetric]( + "numDeletedFiles" -> createMetric(spark.sparkContext, "number of files deleted."), + "numVacuumedDirectories" -> + createMetric(spark.sparkContext, "num of directories vacuumed."), + "status" -> createMetric(spark.sparkContext, "status of vacuum") + ) + metrics("numDeletedFiles").set(filesDeleted.get) + metrics("numVacuumedDirectories").set(dirCounts.get) + txn.registerSQLMetrics(spark, metrics) + } + txn.commit(actions = Seq(), DeltaOperations.VacuumEnd( + status + )) + } + + if (filesDeleted.nonEmpty) { + logConsole(s"Deleted ${filesDeleted.get} files and directories in a total " + + s"of ${dirCounts.get} directories.") + } + } + + /** + * Attempts to relativize the `path` with respect to the `reservoirBase` and converts the path to + * a string. + */ + protected def relativize( + path: Path, + fs: FileSystem, + reservoirBase: Path, + isDir: Boolean): String = { + pathToString(DeltaFileOperations.tryRelativizePath(fs, reservoirBase, path)) + } + + /** + * Wrapper function for DeltaFileOperations.getAllSubDirectories + * returns all subdirectories that `file` has with respect to `base`. + */ + protected def getAllSubdirs(base: String, file: String, fs: FileSystem): Iterator[String] = { + DeltaFileOperations.getAllSubDirectories(base, file)._1 + } + + /** + * Attempts to delete the list of candidate files. Returns the number of files deleted. + */ + protected def delete( + diff: Dataset[String], + spark: SparkSession, + basePath: String, + hadoopConf: Broadcast[SerializableConfiguration], + parallel: Boolean, + parallelPartitions: Int): Long = { + import org.apache.spark.sql.delta.implicits._ + + if (parallel) { + diff.repartition(parallelPartitions).mapPartitions { files => + val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) + val filesDeletedPerPartition = + files.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) + Iterator(filesDeletedPerPartition) + }.collect().sum + } else { + val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) + val fileResultSet = diff.toLocalIterator().asScala + fileResultSet.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) + } + } + + protected def stringToPath(path: String): Path = new Path(new URI(path)) + + protected def pathToString(path: Path): String = path.toUri.toString + + /** Returns the relative path of a file action or None if the file lives outside of the table. */ + protected def getActionRelativePath( + action: FileAction, + fs: FileSystem, + basePath: Path, + relativizeIgnoreError: Boolean): Option[String] = { + val filePath = stringToPath(action.path) + if (filePath.isAbsolute) { + val maybeRelative = + DeltaFileOperations.tryRelativizePath(fs, basePath, filePath, relativizeIgnoreError) + if (maybeRelative.isAbsolute) { + // This file lives outside the directory of the table. + None + } else { + Some(pathToString(maybeRelative)) + } + } else { + Some(pathToString(filePath)) + } + } + + + /** + * Returns the relative paths of all files and subdirectories for this action that must be + * retained during GC. + */ + protected def getValidRelativePathsAndSubdirs( + action: FileAction, + fs: FileSystem, + basePath: Path, + relativizeIgnoreError: Boolean, + isBloomFiltered: Boolean): Seq[String] = { + getActionRelativePath(action, fs, basePath, relativizeIgnoreError).map { relativePath => + Seq(relativePath) ++ getAllSubdirs("/", relativePath, fs) + }.getOrElse(Seq.empty) + } +} + +case class DeltaVacuumStats( + isDryRun: Boolean, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + specifiedRetentionMillis: Option[Long], + defaultRetentionMillis: Long, + minRetainedTimestamp: Long, + dirsPresentBeforeDelete: Long, + objectsDeleted: Long, + sizeOfDataToDelete: Long, + timeTakenToIdentifyEligibleFiles: Long, + timeTakenForDelete: Long) diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala similarity index 68% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala index 52fb921ad486..0a10d073e5c4 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala @@ -14,8 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.spark.sql.delta.files +// scalastyle:off import.ordering.noEmptyLine +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext} import org.apache.spark.internal.Logging import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage @@ -26,18 +30,28 @@ import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_LOCATION, CDC_PART import org.apache.spark.sql.delta.util.{DateFormatter, PartitionUtils, TimestampFormatter, Utils => DeltaUtils} import org.apache.spark.sql.types.StringType -// scalastyle:off import.ordering.noEmptyLine -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext} - import java.util.UUID - import scala.collection.mutable.ArrayBuffer -class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: Option[Int]) - extends FileCommitProtocol - with Serializable - with Logging { +/** + * This file is copied from the DelayedCommitProtocol of the Delta 2.3.0 + * and renamed to MergeTreeCommitProtocol. + * It is modified to overcome the following issues: + * 1. the function commitTask will return TaskCommitMessage(Nil), + * the FileStatus list will be get from the CH backend. + */ + +/** + * Writes out the files to `path` and returns a list of them in `addedStatuses`. Includes + * special handling for partitioning on [[CDC_PARTITION_COL]] for + * compatibility between enabled and disabled CDC; partitions with a value of false in this + * column produce no corresponding partitioning directory. + */ +class MergeTreeCommitProtocol( + jobId: String, + path: String, + randomPrefixLength: Option[Int]) + extends FileCommitProtocol with Serializable with Logging { // Track the list of files added by a task, only used on the executors. @transient protected var addedFiles: ArrayBuffer[(Map[String, String], String)] = _ @@ -58,11 +72,13 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O // Constants for CDC partition manipulation. Used only in newTaskTempFile(), but we define them // here to avoid building a new redundant regex for every file. - protected val cdcPartitionFalse = s"$CDC_PARTITION_COL=false" - protected val cdcPartitionTrue = s"$CDC_PARTITION_COL=true" + protected val cdcPartitionFalse = s"${CDC_PARTITION_COL}=false" + protected val cdcPartitionTrue = s"${CDC_PARTITION_COL}=true" protected val cdcPartitionTrueRegex = cdcPartitionTrue.r - override def setupJob(jobContext: JobContext): Unit = {} + override def setupJob(jobContext: JobContext): Unit = { + + } /** * Commits a job after the writes succeed. Must be called on the driver. Partitions the written @@ -70,8 +86,7 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O * by [[TransactionalWrite]] (i.e. AddFile's may have additional statistics injected) */ override def commitJob(jobContext: JobContext, taskCommits: Seq[TaskCommitMessage]): Unit = { - val (addFiles, changeFiles) = taskCommits - .flatMap(_.obj.asInstanceOf[Seq[_]]) + val (addFiles, changeFiles) = taskCommits.flatMap(_.obj.asInstanceOf[Seq[_]]) .partition { case _: AddFile => true case _: AddCDCFile => false @@ -128,12 +143,14 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O timestampFormatter) ._1 .get - parsedPartition.columnNames - .zip( - parsedPartition.literals - .map(l => Cast(l, StringType).eval()) - .map(Option(_).map(_.toString).orNull)) - .toMap + parsedPartition + .columnNames + .zip( + parsedPartition + .literals + .map(l => Cast(l, StringType).eval()) + .map(Option(_).map(_.toString).orNull)) + .toMap } /** @@ -142,56 +159,46 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O * * Includes special logic for CDC files and paths. Specifically, if the directory `dir` contains * the CDC partition `__is_cdc=true` then - * - the file name begins with `cdc-` instead of `part-` - * - the directory has the `__is_cdc=true` partition removed and is placed in the - * `_changed_data` folder + * - the file name begins with `cdc-` instead of `part-` + * - the directory has the `__is_cdc=true` partition removed and is placed in the `_changed_data` + * folder */ override def newTaskTempFile( - taskContext: TaskAttemptContext, - dir: Option[String], - ext: String): String = { + taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = { val partitionValues = dir.map(parsePartitions).getOrElse(Map.empty[String, String]) val filename = getFileName(taskContext, ext, partitionValues) - val relativePath = randomPrefixLength - .map { - prefixLength => - DeltaUtils.getRandomPrefix(prefixLength) // Generate a random prefix as a first choice - } - .orElse { - dir // or else write into the partition directory if it is partitioned + val relativePath = randomPrefixLength.map { prefixLength => + DeltaUtils.getRandomPrefix(prefixLength) // Generate a random prefix as a first choice + }.orElse { + dir // or else write into the partition directory if it is partitioned + }.map { subDir => + // Do some surgery on the paths we write out to eliminate the CDC_PARTITION_COL. Non-CDC + // data is written to the base location, while CDC data is written to a special folder + // _change_data. + // The code here gets a bit complicated to accommodate two corner cases: an empty subdir + // can't be passed to new Path() at all, and a single-level subdir won't have a trailing + // slash. + if (subDir == cdcPartitionFalse) { + new Path(filename) + } else if (subDir.startsWith(cdcPartitionTrue)) { + val cleanedSubDir = cdcPartitionTrueRegex.replaceFirstIn(subDir, CDC_LOCATION) + new Path(cleanedSubDir, filename) + } else if (subDir.startsWith(cdcPartitionFalse)) { + // We need to remove the trailing slash in addition to the directory - otherwise + // it'll be interpreted as an absolute path and fail. + val cleanedSubDir = subDir.stripPrefix(cdcPartitionFalse + "/") + new Path(cleanedSubDir, filename) + } else { + new Path(subDir, filename) } - .map { - subDir => - // Do some surgery on the paths we write out to eliminate the CDC_PARTITION_COL. Non-CDC - // data is written to the base location, while CDC data is written to a special folder - // _change_data. - // The code here gets a bit complicated to accommodate two corner cases: an empty subdir - // can't be passed to new Path() at all, and a single-level subdir won't have a trailing - // slash. - if (subDir == cdcPartitionFalse) { - new Path(filename) - } else if (subDir.startsWith(cdcPartitionTrue)) { - val cleanedSubDir = cdcPartitionTrueRegex.replaceFirstIn(subDir, CDC_LOCATION) - new Path(cleanedSubDir, filename) - } else if (subDir.startsWith(cdcPartitionFalse)) { - // We need to remove the trailing slash in addition to the directory - otherwise - // it'll be interpreted as an absolute path and fail. - val cleanedSubDir = subDir.stripPrefix(cdcPartitionFalse + "/") - new Path(cleanedSubDir, filename) - } else { - new Path(subDir, filename) - } - } - .getOrElse(new Path(filename)) // or directly write out to the output path + }.getOrElse(new Path(filename)) // or directly write out to the output path addedFiles.append((partitionValues, relativePath.toUri.toString)) new Path(path, relativePath).toString } override def newTaskTempFileAbsPath( - taskContext: TaskAttemptContext, - absoluteDir: String, - ext: String): String = { + taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = { throw DeltaErrors.unsupportedAbsPathAddFile(s"$this") } @@ -213,8 +220,9 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O } override def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage = { - if (addedFiles.nonEmpty) { - /* val fs = new Path(path, addedFiles.head._2).getFileSystem(taskContext.getConfiguration) + // --- modified start + /* if (addedFiles.nonEmpty) { + val fs = new Path(path, addedFiles.head._2).getFileSystem(taskContext.getConfiguration) val statuses: Seq[FileAction] = addedFiles.map { f => val filePath = new Path(path, new Path(new URI(f._2))) val stat = fs.getFileStatus(filePath) @@ -222,15 +230,15 @@ class MergeTreeCommitProtocol(jobId: String, path: String, randomPrefixLength: O buildActionFromAddedFile(f, stat, taskContext) }.toSeq - new TaskCommitMessage(statuses) */ - new TaskCommitMessage(Nil) + new TaskCommitMessage(statuses) } else { new TaskCommitMessage(Nil) - } + } */ + // --- modified end + new TaskCommitMessage(Nil) } override def abortTask(taskContext: TaskAttemptContext): Unit = { // TODO: we can also try delete the addedFiles as a best-effort cleanup. } - } diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala diff --git a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 94% rename from backends-clickhouse/src/main/delta-22/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala index 130790308db8..44e775b66688 100644 --- a/backends-clickhouse/src/main/delta-22/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.types.StructType import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) class DeltaMergeTreeFileFormat(metadata: Metadata) extends DeltaParquetFileFormat(metadata) { protected var database = "" @@ -75,7 +76,10 @@ class DeltaMergeTreeFileFormat(metadata: Metadata) extends DeltaParquetFileForma override def equals(other: Any): Boolean = { other match { case ff: DeltaMergeTreeFileFormat => - ff.columnMappingMode == columnMappingMode && ff.referenceSchema == referenceSchema + ff.columnMappingMode == columnMappingMode && + ff.referenceSchema == referenceSchema && + ff.isSplittable == isSplittable && + ff.disablePushDowns == disablePushDowns case _ => false } } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index 0aab14b78c87..a010a986cd4c 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -69,8 +69,6 @@ class CHTransformerApi extends TransformerApi with Logging { disableBucketedScan, filterExprs ) - case _: TahoeFileIndex => - throw new UnsupportedOperationException("Does not support delta-parquet") case _ => // Generate FilePartition for Parquet CHInputPartitionsUtil( diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index e31560259720..d59467e11293 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -27,10 +27,11 @@ import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} import org.apache.spark.sql.delta.files.MergeTreeCommitProtocol import org.apache.spark.sql.delta.schema.InvariantViolationException import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.execution.SQLExecution +import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker} import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat import org.apache.spark.util.{Clock, SerializableConfiguration} @@ -51,156 +52,146 @@ class ClickhouseOptimisticTransaction( ) } + def insertFakeRowAdaptor(queryPlan: SparkPlan): SparkPlan = queryPlan match { + // if the child is columnar, we can just wrap&transfer the columnar data + case c2r: ColumnarToRowExecBase => + FakeRowAdaptor(c2r.child) + // If the child is aqe, we make aqe "support columnar", + // then aqe itself will guarantee to generate columnar outputs. + // So FakeRowAdaptor will always consumes columnar data, + // thus avoiding the case of c2r->aqe->r2c->writer + case aqe: AdaptiveSparkPlanExec => + FakeRowAdaptor( + AdaptiveSparkPlanExec( + aqe.inputPlan, + aqe.context, + aqe.preprocessingRules, + aqe.isSubquery, + supportsColumnar = true + )) + case other => FakeRowAdaptor(other) + } + override def writeFiles( inputData: Dataset[_], writeOptions: Option[DeltaOptions], additionalConstraints: Seq[Constraint]): Seq[FileAction] = { - hasWritten = true - - val spark = inputData.sparkSession - val (data, partitionSchema) = performCDCPartition(inputData) - val outputPath = deltaLog.dataPath - - val (queryExecution, output, generatedColumnConstraints, _) = - normalizeData(deltaLog, data) - val partitioningColumns = getPartitioningColumns(partitionSchema, output) - - val committer = new MergeTreeCommitProtocol("delta-mergetree", outputPath.toString, None) - - // val (optionalStatsTracker, _) = getOptionalStatsTrackerAndStatsCollection(output, outputPath, - // partitionSchema, data) - val (optionalStatsTracker, _) = (None, None) - - val constraints = - Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints - - SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { - val outputSpec = FileFormatWriter.OutputSpec(outputPath.toString, Map.empty, output) - - val queryPlan = queryExecution.executedPlan - val newQueryPlan = queryPlan match { - // if the child is columnar, we can just wrap&transfer the columnar data - case c2r: ColumnarToRowExecBase => - FakeRowAdaptor(c2r.child) - // If the child is aqe, we make aqe "support columnar", - // then aqe itself will guarantee to generate columnar outputs. - // So FakeRowAdaptor will always consumes columnar data, - // thus avoiding the case of c2r->aqe->r2c->writer - case aqe: AdaptiveSparkPlanExec => - FakeRowAdaptor( - AdaptiveSparkPlanExec( - aqe.inputPlan, - aqe.context, - aqe.preprocessingRules, - aqe.isSubquery, - supportsColumnar = true - )) - case other => FakeRowAdaptor(other) - } - - val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() - - if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { - val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( - new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), - BasicWriteJobStatsTracker.metrics) -// registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) - statsTrackers.append(basicWriteJobStatsTracker) - } - - // Retain only a minimal selection of Spark writer options to avoid any potential - // compatibility issues - var options = writeOptions match { - case None => Map.empty[String, String] - case Some(writeOptions) => - writeOptions.options.filterKeys { - key => - key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || - key.equalsIgnoreCase(DeltaOptions.COMPRESSION) - }.toMap - } - - spark.conf.getAll.foreach( - entry => { - if ( - entry._1.startsWith(s"${CHBackendSettings.getBackendConfigPrefix}.runtime_settings") - || entry._1.equalsIgnoreCase(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key) - ) { - options += (entry._1 -> entry._2) - } - }) - - try { - val tableV2 = ClickHouseTableV2.getTable(deltaLog) - MergeTreeFileFormatWriter.write( - sparkSession = spark, - plan = newQueryPlan, - fileFormat = new DeltaMergeTreeFileFormat( - metadata, - tableV2.dataBaseName, - tableV2.tableName, - ClickhouseSnapshot.genSnapshotId(tableV2.snapshot), - tableV2.orderByKeyOption, - tableV2.lowCardKeyOption, - tableV2.minmaxIndexKeyOption, - tableV2.bfIndexKeyOption, - tableV2.setIndexKeyOption, - tableV2.primaryKeyOption, - tableV2.clickhouseTableConfigs, - tableV2.partitionColumns - ), - // formats. - committer = committer, - outputSpec = outputSpec, - // scalastyle:off deltahadoopconfiguration - hadoopConf = - spark.sessionState.newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), - // scalastyle:on deltahadoopconfiguration - orderByKeyOption = tableV2.orderByKeyOption, - lowCardKeyOption = tableV2.lowCardKeyOption, - minmaxIndexKeyOption = tableV2.minmaxIndexKeyOption, - bfIndexKeyOption = tableV2.bfIndexKeyOption, - setIndexKeyOption = tableV2.setIndexKeyOption, - primaryKeyOption = tableV2.primaryKeyOption, - partitionColumns = partitioningColumns, - bucketSpec = tableV2.bucketOption, - statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers, - options = options, - constraints = constraints - ) - } catch { - case s: SparkException => - // Pull an InvariantViolationException up to the top level if it was the root cause. - val violationException = ExceptionUtils.getRootCause(s) - if (violationException.isInstanceOf[InvariantViolationException]) { - throw violationException - } else { - throw s - } + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + hasWritten = true + + val spark = inputData.sparkSession + val (data, partitionSchema) = performCDCPartition(inputData) + val outputPath = deltaLog.dataPath + + val (queryExecution, output, generatedColumnConstraints, _) = + normalizeData(deltaLog, data) + val partitioningColumns = getPartitioningColumns(partitionSchema, output) + + val committer = new MergeTreeCommitProtocol("delta-mergetree", outputPath.toString, None) + + // val (optionalStatsTracker, _) = + // getOptionalStatsTrackerAndStatsCollection(output, outputPath, partitionSchema, data) + val (optionalStatsTracker, _) = (None, None) + + val constraints = + Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints + + SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { + val outputSpec = FileFormatWriter.OutputSpec(outputPath.toString, Map.empty, output) + + val queryPlan = queryExecution.executedPlan + val newQueryPlan = insertFakeRowAdaptor(queryPlan) + + val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() + + if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { + val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( + new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), + BasicWriteJobStatsTracker.metrics) + // registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) + statsTrackers.append(basicWriteJobStatsTracker) + } + + // Retain only a minimal selection of Spark writer options to avoid any potential + // compatibility issues + var options = writeOptions match { + case None => Map.empty[String, String] + case Some(writeOptions) => + writeOptions.options.filterKeys { + key => + key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || + key.equalsIgnoreCase(DeltaOptions.COMPRESSION) + }.toMap + } + + spark.conf.getAll.foreach( + entry => { + if ( + entry._1.startsWith(s"${CHBackendSettings.getBackendConfigPrefix}.runtime_settings") + || entry._1.equalsIgnoreCase(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key) + ) { + options += (entry._1 -> entry._2) + } + }) + + try { + val tableV2 = ClickHouseTableV2.getTable(deltaLog) + MergeTreeFileFormatWriter.write( + sparkSession = spark, + plan = newQueryPlan, + fileFormat = new DeltaMergeTreeFileFormat( + metadata, + tableV2.dataBaseName, + tableV2.tableName, + ClickhouseSnapshot.genSnapshotId(tableV2.snapshot), + tableV2.orderByKeyOption, + tableV2.lowCardKeyOption, + tableV2.minmaxIndexKeyOption, + tableV2.bfIndexKeyOption, + tableV2.setIndexKeyOption, + tableV2.primaryKeyOption, + tableV2.clickhouseTableConfigs, + tableV2.partitionColumns + ), + // formats. + committer = committer, + outputSpec = outputSpec, + // scalastyle:off deltahadoopconfiguration + hadoopConf = spark.sessionState + .newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), + // scalastyle:on deltahadoopconfiguration + orderByKeyOption = tableV2.orderByKeyOption, + lowCardKeyOption = tableV2.lowCardKeyOption, + minmaxIndexKeyOption = tableV2.minmaxIndexKeyOption, + bfIndexKeyOption = tableV2.bfIndexKeyOption, + setIndexKeyOption = tableV2.setIndexKeyOption, + primaryKeyOption = tableV2.primaryKeyOption, + partitionColumns = partitioningColumns, + bucketSpec = tableV2.bucketOption, + statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers, + options = options, + constraints = constraints + ) + } catch { + case s: SparkException => + // Pull an InvariantViolationException up to the top level if it was the root cause. + val violationException = ExceptionUtils.getRootCause(s) + if (violationException.isInstanceOf[InvariantViolationException]) { + throw violationException + } else { + throw s + } + } } + committer.addedStatuses.toSeq ++ committer.changeFiles + } else { + // TODO: support native delta parquet write + // 1. insert FakeRowAdaptor + // 2. DeltaInvariantCheckerExec transform + // 3. DeltaTaskStatisticsTracker collect null count / min values / max values + // 4. set the parameters 'staticPartitionWriteOnly', 'isNativeAppliable', + // 'nativeFormat' in the LocalProperty of the sparkcontext + super.writeFiles(inputData, writeOptions, additionalConstraints) } - - // val resultFiles = committer.addedStatuses - // .map { - // a => - // a.copy(stats = optionalStatsTracker - // .map(_.recordedStats(new Path(new URI(a.path)).getName)) - // .getOrElse(a.stats)) - // } - /* - .filter { - // In some cases, we can write out an empty `inputData`. - // Some examples of this (though, they - // may be fixed in the future) are the MERGE command when you delete with empty source, or - // empty target, or on disjoint tables. This is hard to catch before the write without - // collecting the DF ahead of time. Instead, we can return only the AddFiles that - // a) actually add rows, or - // b) don't have any stats so we don't know the number of rows at all - case a: AddFile => a.numLogicalRecords.forall(_ > 0) - case _ => true - } - */ - - committer.addedStatuses.toSeq ++ committer.changeFiles } } diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 1107c6a2ef76..d5cd4f984ca6 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -19,7 +19,6 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.connector.catalog.TableCatalog import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTimeTravelSpec} @@ -28,7 +27,6 @@ import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table import org.apache.spark.sql.delta.sources.DeltaDataSource import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} import org.apache.spark.sql.execution.datasources.utils.MergeTreePartsPartitionsUtil -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.collection.BitSet @@ -83,7 +81,6 @@ class ClickHouseTableV2( override def properties(): ju.Map[String, String] = { val ret = super.properties() - ret.put(TableCatalog.PROP_PROVIDER, ClickHouseConfig.NAME) // for file path based write if (snapshot.version < 0 && clickhouseExtensionOptions.nonEmpty) { @@ -232,6 +229,7 @@ class ClickHouseTableV2( partitionColumns ) } + def cacheThis(): Unit = { deltaLog2Table.put(deltaLog, this) } diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala index 7af1abe43722..f7a180b6a239 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala @@ -25,6 +25,7 @@ import org.apache.spark.internal.io.SparkHadoopWriterUtils import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 @@ -243,21 +244,6 @@ object OptimizeTableCommandOverwrites extends Logging { } - private def isDeltaTable(spark: SparkSession, tableName: TableIdentifier): Boolean = { - val catalog = spark.sessionState.catalog - val tableIsNotTemporaryTable = !catalog.isTempView(tableName) - val tableExists = { - (tableName.database.isEmpty || catalog.databaseExists(tableName.database.get)) && - catalog.tableExists(tableName) - } - tableIsNotTemporaryTable && tableExists && catalog - .getTableMetadata(tableName) - .provider - .get - .toLowerCase() - .equals("clickhouse") - } - def getDeltaLogClickhouse( spark: SparkSession, path: Option[String], @@ -276,7 +262,17 @@ object OptimizeTableCommandOverwrites extends Logging { } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) { new Path(metadata.location) } else { - throw DeltaErrors.notADeltaTableException(operationName) + DeltaTableIdentifier(spark, tableIdentifier.get) match { + case Some(id) if id.path.nonEmpty => + new Path(id.path.get) + case Some(id) if id.table.nonEmpty => + new Path(metadata.location) + case _ => + if (metadata.tableType == CatalogTableType.VIEW) { + throw DeltaErrors.viewNotSupported(operationName) + } + throw DeltaErrors.notADeltaTableException(operationName) + } } } else { throw DeltaErrors.missingTableIdentifierException(operationName) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala index e7eb3aeb05d0..232e9ec10c5b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.execution.datasources.v2.clickhouse import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.connector.catalog.TableCatalog import java.util @@ -29,7 +28,8 @@ object ClickHouseConfig { val NAME = "clickhouse" val ALT_NAME = "clickhouse" val METADATA_DIR = "_delta_log" - val DEFAULT_ENGINE = "MergeTree" + val FORMAT_ENGINE = "engine" + val DEFAULT_ENGINE = "mergetree" val OPT_NAME_PREFIX = "clickhouse." @deprecated @@ -47,16 +47,15 @@ object ClickHouseConfig { buckets: Option[BucketSpec] = None): Map[String, String] = { val configurations = scala.collection.mutable.Map[String, String]() allProperties.asScala.foreach(configurations += _) - configurations.put(TableCatalog.PROP_PROVIDER, ClickHouseConfig.NAME) if (!configurations.contains("metadata_path")) { configurations += ("metadata_path" -> METADATA_DIR) } - if (!configurations.contains("engine")) { - configurations += ("engine" -> DEFAULT_ENGINE) + if (!configurations.contains(FORMAT_ENGINE)) { + configurations += (FORMAT_ENGINE -> DEFAULT_ENGINE) } else { - val engineValue = configurations.get("engine") + val engineValue = configurations.get(FORMAT_ENGINE) if (!engineValue.equals(DEFAULT_ENGINE) && !engineValue.equals("parquet")) { - configurations += ("engine" -> DEFAULT_ENGINE) + configurations += (FORMAT_ENGINE -> DEFAULT_ENGINE) } } if (!configurations.contains("sampling_key")) { @@ -80,6 +79,11 @@ object ClickHouseConfig { configurations.toMap } + def isMergeTreeFormatEngine(configuration: Map[String, String]): Boolean = { + configuration.contains(FORMAT_ENGINE) && + configuration.get(FORMAT_ENGINE).get.equals(DEFAULT_ENGINE) + } + /** Get the related clickhouse option when using DataFrameWriter / DataFrameReader */ def getMergeTreeConfigurations( properties: util.Map[String, String] diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala index c2d208df5ed9..61e1da44d0af 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.spark.sql.execution.datasources.v2.clickhouse + import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession} @@ -26,12 +27,12 @@ import org.apache.spark.sql.connector.catalog._ import org.apache.spark.sql.connector.catalog.TableCapability.V1_BATCH_WRITE import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} -import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaOptions} +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, TempClickHouseTableV2} +import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils import org.apache.spark.sql.sources.InsertableRelation @@ -52,6 +53,15 @@ class ClickHouseSparkCatalog val spark = SparkSession.active + private def createCatalogTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String] + ): Table = { + super.createTable(ident, schema, partitions, properties) + } + override def createTable( ident: Identifier, schema: StructType, @@ -66,8 +76,18 @@ class ClickHouseSparkCatalog Map.empty, sourceQuery = None, TableCreationModes.Create) + } else if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { + createDeltaTable( + ident, + schema, + partitions, + properties, + Map.empty, + sourceQuery = None, + TableCreationModes.Create + ) } else { - super.createTable(ident, schema, partitions, properties) + createCatalogTable(ident, schema, partitions, properties) } } @@ -120,7 +140,10 @@ class ClickHouseSparkCatalog .copy(locationUri = locUriOpt) val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED - val id = TableIdentifier(ident.name(), ident.namespace().lastOption) + val id = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + val existingTableOpt = getExistingTableIfExists(id) val loc = new Path(locUriOpt.getOrElse(spark.sessionState.catalog.defaultTablePath(id))) val commentOpt = Option(allTableProperties.get("comment")) @@ -136,7 +159,7 @@ class ClickHouseSparkCatalog comment = commentOpt ) - val withDb = verifyTableAndSolidify(tableDesc, None) + val withDb = verifyTableAndSolidify(tableDesc, None, true) val writer = sourceQuery.map { df => @@ -156,7 +179,7 @@ class ClickHouseSparkCatalog CreateDeltaTableCommand( withDb, - getExistingTableIfExists(tableDesc), + existingTableOpt, operation.mode, writer, operation = operation, @@ -166,14 +189,134 @@ class ClickHouseSparkCatalog } logInfo(s"create table ${ident.toString} successfully.") - val loadedNewTable = loadTable(ident) - loadedNewTable + loadTable(ident) + } + + /** + * Creates a Delta table + * + * @param ident + * The identifier of the table + * @param schema + * The schema of the table + * @param partitions + * The partition transforms for the table + * @param allTableProperties + * The table properties that configure the behavior of the table or provide information about + * the table + * @param writeOptions + * Options specific to the write during table creation or replacement + * @param sourceQuery + * A query if this CREATE request came from a CTAS or RTAS + * @param operation + * The specific table creation mode, whether this is a Create/Replace/Create or Replace + */ + private def createDeltaTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + allTableProperties: util.Map[String, String], + writeOptions: Map[String, String], + sourceQuery: Option[DataFrame], + operation: TableCreationModes.CreationMode + ): Table = { + // These two keys are tableProperties in data source v2 but not in v1, so we have to filter + // them out. Otherwise property consistency checks will fail. + val tableProperties = allTableProperties.asScala.filterKeys { + case TableCatalog.PROP_LOCATION => false + case TableCatalog.PROP_PROVIDER => false + case TableCatalog.PROP_COMMENT => false + case TableCatalog.PROP_OWNER => false + case TableCatalog.PROP_EXTERNAL => false + case "path" => false + case _ => true + }.toMap + val (partitionColumns, maybeBucketSpec) = + SparkShimLoader.getSparkShims.convertPartitionTransforms(partitions) + var newSchema = schema + var newPartitionColumns = partitionColumns + var newBucketSpec = maybeBucketSpec + val conf = spark.sessionState.conf + + val isByPath = isPathIdentifier(ident) + if ( + isByPath && !conf.getConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS) + && allTableProperties.containsKey("location") + // The location property can be qualified and different from the path in the identifier, so + // we check `endsWith` here. + && Option(allTableProperties.get("location")).exists(!_.endsWith(ident.name())) + ) { + throw DeltaErrors.ambiguousPathsInCreateTableException( + ident.name(), + allTableProperties.get("location")) + } + val location = if (isByPath) { + Option(ident.name()) + } else { + Option(allTableProperties.get("location")) + } + val id = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + var locUriOpt = location.map(CatalogUtils.stringToURI) + val existingTableOpt = getExistingTableIfExists(id) + val loc = locUriOpt + .orElse(existingTableOpt.flatMap(_.storage.locationUri)) + .getOrElse(spark.sessionState.catalog.defaultTablePath(id)) + val storage = DataSource + .buildStorageFormatFromOptions(writeOptions) + .copy(locationUri = Option(loc)) + val tableType = + if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED + val commentOpt = Option(allTableProperties.get("comment")) + + var tableDesc = new CatalogTable( + identifier = id, + tableType = tableType, + storage = storage, + schema = newSchema, + provider = Some(DeltaSourceUtils.ALT_NAME), + partitionColumnNames = newPartitionColumns, + bucketSpec = newBucketSpec, + properties = tableProperties, + comment = commentOpt + ) + + val withDb = verifyTableAndSolidify(tableDesc, None) + + val writer = sourceQuery.map { + df => + WriteIntoDelta( + DeltaLog.forTable(spark, new Path(loc)), + operation.mode, + new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), + withDb.partitionColumnNames, + withDb.properties ++ commentOpt.map("comment" -> _), + df, + schemaInCatalog = if (newSchema != schema) Some(newSchema) else None + ) + } + + CreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation, + tableByPath = isByPath).run(spark) + + loadTable(ident) } /** Performs checks on the parameters provided for table creation for a ClickHouse table. */ private def verifyTableAndSolidify( tableDesc: CatalogTable, - query: Option[LogicalPlan]): CatalogTable = { + query: Option[LogicalPlan], + isMergeTree: Boolean = false): CatalogTable = { + + if (!isMergeTree && tableDesc.bucketSpec.isDefined) { + throw DeltaErrors.operationNotSupportedException("Bucketing", tableDesc.identifier) + } val schema = query .map { @@ -189,30 +332,36 @@ class ClickHouseSparkCatalog caseSensitive = false ) // Delta is case insensitive + val validatedConfigurations = if (isMergeTree) { + tableDesc.properties + } else { + DeltaConfigs.validateConfigurations(tableDesc.properties) + } + val db = tableDesc.identifier.database.getOrElse(catalog.getCurrentDatabase) val tableIdentWithDB = tableDesc.identifier.copy(database = Some(db)) tableDesc.copy( identifier = tableIdentWithDB, schema = schema, - properties = tableDesc.properties) + properties = validatedConfigurations) } /** Checks if a table already exists for the provided identifier. */ - private def getExistingTableIfExists(table: CatalogTable): Option[CatalogTable] = { + def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { // If this is a path identifier, we cannot return an existing CatalogTable. The Create command // will check the file system itself if (isPathIdentifier(table)) return None - val tableExists = catalog.tableExists(table.identifier) + val tableExists = catalog.tableExists(table) if (tableExists) { - val oldTable = catalog.getTableMetadata(table.identifier) + val oldTable = catalog.getTableMetadata(table) if (oldTable.tableType == CatalogTableType.VIEW) { - throw new AnalysisException( - s"${table.identifier} is a view. You may not write data into a view.") + throw new AnalysisException(s"$table is a view. You may not write data into a view.") } - if (!CHDataSourceUtils.isClickHouseTable(oldTable.provider)) { - throw new AnalysisException( - s"${table.identifier} is not a ClickHouse table. Please drop " + - s"this table first if you would like to recreate it.") + if ( + !DeltaSourceUtils.isDeltaTable(oldTable.provider) && + !CHDataSourceUtils.isClickHouseTable(oldTable.provider) + ) { + throw DeltaErrors.notADeltaTable(table.table) } Some(oldTable) } else { @@ -233,6 +382,12 @@ class ClickHouseSparkCatalog new Path(v1.catalogTable.location), catalogTable = Some(v1.catalogTable), tableIdentifier = Some(ident.toString)) + case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => + DeltaTableV2( + spark, + new Path(v1.catalogTable.location), + catalogTable = Some(v1.catalogTable), + tableIdentifier = Some(ident.toString)) case o => o } @@ -249,8 +404,12 @@ class ClickHouseSparkCatalog } } - private def newDeltaPathTable(ident: Identifier): ClickHouseTableV2 = { - new ClickHouseTableV2(spark, new Path(ident.name())) + private def newDeltaPathTable(ident: Identifier): DeltaTableV2 = { + if (hasClickHouseNamespace(ident)) { + new ClickHouseTableV2(spark, new Path(ident.name())) + } else { + DeltaTableV2(spark, new Path(ident.name())) + } } /** support to delete mergetree data from the external table */ @@ -284,11 +443,15 @@ class ClickHouseSparkCatalog partitions: Array[Transform], properties: util.Map[String, String]): StagedTable = recordFrameProfile("DeltaCatalog", "stageReplace") { - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Replace) } else { super.dropTable(ident) - BestEffortStagedTable(ident, super.createTable(ident, schema, partitions, properties), this) + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) } } @@ -298,7 +461,10 @@ class ClickHouseSparkCatalog partitions: Array[Transform], properties: util.Map[String, String]): StagedTable = recordFrameProfile("DeltaCatalog", "stageCreateOrReplace") { - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { new StagedDeltaTableV2( ident, schema, @@ -311,7 +477,8 @@ class ClickHouseSparkCatalog case _: NoSuchDatabaseException => // this is fine case _: NoSuchTableException => // this is fine } - BestEffortStagedTable(ident, super.createTable(ident, schema, partitions, properties), this) + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) } } @@ -321,13 +488,22 @@ class ClickHouseSparkCatalog partitions: Array[Transform], properties: util.Map[String, String]): StagedTable = recordFrameProfile("DeltaCatalog", "stageCreate") { - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Create) } else { - BestEffortStagedTable(ident, super.createTable(ident, schema, partitions, properties), this) + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) } } + /** + * A staged delta table, which creates a HiveMetaStore entry and appends data if this was a + * CTAS/RTAS command. We have a ugly way of using this API right now, but it's the best way to + * maintain old behavior compatibility between Databricks Runtime and OSS Delta Lake. + */ private class StagedDeltaTableV2( ident: Identifier, override val schema: StructType, @@ -374,14 +550,18 @@ class ClickHouseSparkCatalog } } } - createClickHouseTable( - ident, - schema, - partitions, - props, - writeOptions, - asSelectQuery, - operation) + if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + createClickHouseTable( + ident, + schema, + partitions, + props, + writeOptions, + asSelectQuery, + operation) + } else { + createDeltaTable(ident, schema, partitions, props, writeOptions, asSelectQuery, operation) + } } override def name(): String = ident.name() @@ -454,20 +634,29 @@ trait SupportsPathIdentifier extends TableCatalog { protected def isPathIdentifier(ident: Identifier): Boolean = { // Should be a simple check of a special PathIdentifier class in the future try { - supportSQLOnFile && hasClickHouseNamespace(ident) && new Path(ident.name()).isAbsolute + supportSQLOnFile && (hasClickHouseNamespace(ident) || hasDeltaNamespace(ident)) && + new Path(ident.name()).isAbsolute } catch { case _: IllegalArgumentException => false } } + protected def isPathIdentifier(table: CatalogTable): Boolean = { + isPathIdentifier(table.identifier) + } + + protected def isPathIdentifier(tableIdentifier: TableIdentifier): Boolean = { + isPathIdentifier(Identifier.of(tableIdentifier.database.toArray, tableIdentifier.table)) + } + private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile - private def hasClickHouseNamespace(ident: Identifier): Boolean = { + protected def hasClickHouseNamespace(ident: Identifier): Boolean = { ident.namespace().length == 1 && CHDataSourceUtils.isClickHouseDataSourceName(ident.namespace().head) } - protected def isPathIdentifier(table: CatalogTable): Boolean = { - isPathIdentifier(Identifier.of(table.identifier.database.toArray, table.identifier.table)) + protected def hasDeltaNamespace(ident: Identifier): Boolean = { + ident.namespace().length == 1 && DeltaSourceUtils.isDeltaDataSourceName(ident.namespace().head) } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala new file mode 100644 index 000000000000..a097fc6cd4ab --- /dev/null +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala @@ -0,0 +1,1430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.gluten.GlutenConfig + +import org.apache.spark.SparkConf +import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.delta.actions.AddFile +import org.apache.spark.sql.delta.files.TahoeFileIndex +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + +import io.delta.tables.DeltaTable + +import java.io.File + +// Some sqls' line length exceeds 100 +// scalastyle:off line.size.limit + +class GlutenClickHouseDeltaParquetWriteSuite + extends GlutenClickHouseTPCHAbstractSuite + with AdaptiveSparkPlanHelper { + + override protected val needCopyParquetToTablePath = true + + override protected val tablesPath: String = basePath + "/tpch-data" + override protected val tpchQueries: String = rootPath + "queries/tpch-queries-ch" + override protected val queriesResults: String = rootPath + "mergetree-queries-output" + + /** Run Gluten + ClickHouse Backend with SortShuffleManager */ + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + .set("spark.io.compression.codec", "LZ4") + .set("spark.sql.shuffle.partitions", "5") + .set("spark.sql.autoBroadcastJoinThreshold", "10MB") + .set("spark.sql.adaptive.enabled", "true") + .set("spark.sql.files.maxPartitionBytes", "20000000") + .set("spark.gluten.sql.native.writer.enabled", "true") + .set("spark.sql.storeAssignmentPolicy", "legacy") + .set( + "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert", + "false") + .set( + "spark.databricks.delta.retentionDurationCheck.enabled", + "false" + ) + } + + override protected def createTPCHNotNullTables(): Unit = { + createNotNullTPCHTablesInParquet(tablesPath) + } + + test("test parquet table write with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet + | select /*+ REPARTITION(5) */ * from lineitem + |""".stripMargin) + + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | lineitem_delta_parquet + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr) { + df => + val plans = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + case w: WholeStageTransformer => w + } + assert(plans.size == 4) + + val parquetScan = plans(3).asInstanceOf[FileSourceScanExecTransformer] + assert(parquetScan.nodeName.startsWith("Scan parquet ")) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 5) + } + } + + test("test parquet insert overwrite with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_insertoverwrite; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_insertoverwrite + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet_insertoverwrite' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet_insertoverwrite + | select * from lineitem + |""".stripMargin) + + spark.sql(s""" + | insert overwrite table lineitem_delta_parquet_insertoverwrite + | select * from lineitem where mod(l_orderkey,2) = 1 + |""".stripMargin) + val sql2 = + s""" + | select count(*) from lineitem_delta_parquet_insertoverwrite + |""".stripMargin + assert( + // total rows should remain unchanged + spark.sql(sql2).collect().apply(0).get(0) == 300001 + ) + } + + test("test parquet insert overwrite partitioned table with small table, static with delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_insertoverwrite2; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_insertoverwrite2 + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |PARTITIONED BY (l_shipdate) + |LOCATION '$basePath/lineitem_delta_parquet_insertoverwrite2' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet_insertoverwrite2 + | select * from lineitem + |""".stripMargin) + + spark.sql( + s""" + | insert overwrite table lineitem_delta_parquet_insertoverwrite2 + | select * from lineitem where l_shipdate BETWEEN date'1993-02-01' AND date'1993-02-10' + |""".stripMargin) + val sql2 = + s""" + | select count(*) from lineitem_delta_parquet_insertoverwrite2 + | + |""".stripMargin + assert( + // total rows should remain unchanged + spark.sql(sql2).collect().apply(0).get(0) == 2418 + ) + } + + test("test parquet insert overwrite partitioned table with small table, dynamic with delta") { + withSQLConf(("spark.sql.sources.partitionOverwriteMode", "dynamic")) { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_insertoverwrite3 PURGE; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_insertoverwrite3 + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |PARTITIONED BY (l_shipdate) + |LOCATION '$basePath/lineitem_delta_parquet_insertoverwrite3' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet_insertoverwrite3 + | select * from lineitem + |""".stripMargin) + + spark.sql( + s""" + | insert overwrite table lineitem_delta_parquet_insertoverwrite3 + | select * from lineitem where l_shipdate BETWEEN date'1993-02-01' AND date'1993-02-10' + |""".stripMargin) + val sql2 = + s""" + | select count(*) from lineitem_delta_parquet_insertoverwrite3 + | + |""".stripMargin + assert( + // total rows should remain unchanged + spark.sql(sql2).collect().apply(0).get(0) == 600572 + ) + } + } + + test("test parquet table update with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_update; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_update + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet_update' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet_update + | select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + + spark.sql( + s""" + | update lineitem_delta_parquet_update set l_returnflag = 'Z' where l_orderkey = 12647 + |""".stripMargin) + + { + val sql1 = + s""" + | select count(*) from lineitem_delta_parquet_update where l_returnflag = 'Z' + | + |""".stripMargin + + val df = spark.sql(sql1) + val result = df.collect() + assert( + // in test data, there are only 1 row with l_orderkey = 12647 + result.apply(0).get(0) == 1 + ) + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val parquetScan = scanExec.head + assert(parquetScan.nodeName.startsWith("Scan parquet")) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 4) + } + + val sql2 = + s""" + | select count(*) from lineitem_delta_parquet_update + | + |""".stripMargin + assert( + // total rows should remain unchanged + spark.sql(sql2).collect().apply(0).get(0) == 600572 + ) + } + + test("test parquet table delete with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_delete; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_delete + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet_delete' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet_delete + | select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + val df1 = spark.sql(s""" + | delete from lineitem_delta_parquet_delete where l_orderkey = 12647 + |""".stripMargin) + + { + val df = spark.sql(s""" + | select sum(l_linenumber) from lineitem_delta_parquet_delete + |""".stripMargin) + val result = df.collect() + assert( + result.apply(0).get(0) == 1802445 + ) + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + val parquetScan = scanExec.head + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 4) + } + + { + spark.sql(s""" + | delete from lineitem_delta_parquet_delete where mod(l_orderkey, 3) = 2 + |""".stripMargin) + val df3 = spark.sql(s""" + | select sum(l_linenumber) from lineitem_delta_parquet_delete + |""".stripMargin) + assert( + df3.collect().apply(0).get(0) == 1200671 + ) + } + } + + test("test parquet table upsert with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_upsert; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_upsert + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet_upsert' + |""".stripMargin) + + spark.sql(s""" + | insert into table lineitem_delta_parquet_upsert + | select * from lineitem + |""".stripMargin) + + { + val df0 = spark.sql(s""" + | select sum(l_linenumber) from lineitem_delta_parquet_upsert + |""".stripMargin) + assert( + df0.collect().apply(0).get(0) == 1802446 + ) + } + + upsertSourceTableAndCheck("lineitem_delta_parquet_upsert") + } + + private def upsertSourceTableAndCheck(tableName: String) = { + // Why selecting l_orderkey having count(*) =1 ? + // Answer: to avoid "org.apache.spark.sql.delta.DeltaUnsupportedOperationException: + // Cannot perform Merge as multiple source rows matched and attempted to modify the same + // target row in the Delta table in possibly conflicting ways." + spark.sql(s""" + merge into $tableName + using ( + + select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, + 'Z' as `l_returnflag`, + l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + from lineitem where l_orderkey in (select l_orderkey from lineitem group by l_orderkey having count(*) =1 ) and l_orderkey < 100000 + + union + + select l_orderkey + 10000000, + l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, + l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + from lineitem where l_orderkey in (select l_orderkey from lineitem group by l_orderkey having count(*) =1 ) and l_orderkey < 100000 + + ) as updates + on updates.l_orderkey = $tableName.l_orderkey + when matched then update set * + when not matched then insert * + """.stripMargin) + + { + val df1 = spark.sql(s""" + | select count(*) from $tableName + |""".stripMargin) + assert( + df1.collect().apply(0).get(0) == 600572 + 3506 + ) + } + { + val df2 = + spark.sql(s""" + | select count(*) from $tableName where l_returnflag = 'Z' + |""".stripMargin) + assert( + df2.collect().apply(0).get(0) == 3506 + ) + } + + { + val df3 = + spark.sql(s""" + | select count(*) from $tableName where l_orderkey > 10000000 + |""".stripMargin) + assert( + df3.collect().apply(0).get(0) == 3506 + ) + } + } + + test("test parquet write with partition + delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_partition; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_partition + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |PARTITIONED BY (l_shipdate, l_returnflag) + |LOCATION '$basePath/lineitem_delta_parquet_partition' + |""".stripMargin) + + // dynamic partitions + spark.sql(s""" + | insert into table lineitem_delta_parquet_partition + | select * from lineitem + |""".stripMargin) + + // write with dataframe api + val source = spark.sql(s""" + |select + | l_orderkey , + | l_partkey , + | l_suppkey , + | l_linenumber , + | l_quantity , + | l_extendedprice , + | l_discount , + | l_tax , + | l_returnflag , + | l_linestatus , + | l_shipdate , + | l_commitdate , + | l_receiptdate , + | l_shipinstruct , + | l_shipmode , + | l_comment + | from lineitem + | where l_shipdate BETWEEN date'1993-01-01' AND date'1993-01-10' + |""".stripMargin) + + source.write + .format("delta") + .mode(SaveMode.Append) + .insertInto("lineitem_delta_parquet_partition") + + // static partition + spark.sql( + s""" + | insert into lineitem_delta_parquet_partition PARTITION (l_shipdate=date'1995-01-21', + | l_returnflag = 'A') + | (l_orderkey, + | l_partkey, + | l_suppkey, + | l_linenumber, + | l_quantity, + | l_extendedprice, + | l_discount, + | l_tax, + | l_linestatus, + | l_commitdate, + | l_receiptdate, + | l_shipinstruct, + | l_shipmode, + | l_comment) + | select l_orderkey, + | l_partkey, + | l_suppkey, + | l_linenumber, + | l_quantity, + | l_extendedprice, + | l_discount, + | l_tax, + | l_linestatus, + | l_commitdate, + | l_receiptdate, + | l_shipinstruct, + | l_shipmode, + | l_comment from lineitem + | where l_shipdate BETWEEN date'1993-02-01' AND date'1993-02-10' + |""".stripMargin) + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | lineitem_delta_parquet_partition + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr, compareResult = false) { + df => + val result = df.collect() + assert(result.size == 4) + assert(result(0).getString(0).equals("A")) + assert(result(0).getString(1).equals("F")) + assert(result(0).getDouble(2) == 3865234.0) + + assert(result(2).getString(0).equals("N")) + assert(result(2).getString(1).equals("O")) + assert(result(2).getDouble(2) == 7454519.0) + + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val parquetScan = scanExec(0) + assert(parquetScan.nodeName.startsWith("Scan parquet")) + assert(parquetScan.metrics("numFiles").value == 3745) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + + assert(addFiles.size == 3836) + assert( + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 2) + assert( + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) + assert( + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 3) + } + } + + test("test parquet CTAS simple with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_ctas1; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE lineitem_delta_parquet_ctas1 + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet_ctas1' + | as select * from lineitem + |""".stripMargin) + + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | lineitem_delta_parquet_ctas1 + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr) { + df => + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val parquetScan = scanExec(0) + assert(parquetScan.nodeName.startsWith("Scan parquet")) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 1) + } + } + + test("test parquet CTAS complex with the delta") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_ctas2; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_ctas2 + |USING delta + |PARTITIONED BY (l_shipdate) + |LOCATION '$basePath/lineitem_mergetree_ctas2' + | as select * from lineitem + |""".stripMargin) + + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | lineitem_delta_parquet_ctas2 + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr) { _ => {} } + + } + + test("test path based parquet write with the delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_filebased" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select * from lineitem + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE delta.`$dataPath` ( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING delta + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + sourceDF.write + .format("delta") + .mode(SaveMode.Overwrite) + .save(dataPath) + + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | delta.`$dataPath` + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr) { + df => + val plans = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + case w: WholeStageTransformer => w + } + assert(plans.size == 4) + + val parquetScan = plans(3).asInstanceOf[FileSourceScanExecTransformer] + assert(parquetScan.nodeName.startsWith("Scan parquet")) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 1) + } + + val result = spark.read + .format("delta") + .load(dataPath) + .where("l_shipdate = date'1998-09-02'") + .count() + assert(result == 183) + } + + test( + "test path based parquet insert overwrite partitioned table with small table, static with delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_insertoverwrite2" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .partitionBy("l_shipdate") + .save(dataPath) + + val sourceDF1 = spark.sql( + s""" + |select * from lineitem where l_shipdate BETWEEN date'1993-02-01' AND date'1993-02-10' + |""".stripMargin) + sourceDF1.write + .format("delta") + .mode(SaveMode.Overwrite) + .partitionBy("l_shipdate") + .save(dataPath) + + val result = spark.read + .format("delta") + .load(dataPath) + .count() + assert(result == 2418) + } + + test( + "test path based parquet insert overwrite partitioned table with small table, dynamic with delta") { + withSQLConf(("spark.sql.sources.partitionOverwriteMode", "dynamic")) { + val dataPath = s"$basePath/lineitem_delta_parquet_insertoverwrite3" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .partitionBy("l_shipdate") + .save(dataPath) + + val sourceDF1 = spark.sql( + s""" + |select * from lineitem where l_shipdate BETWEEN date'1993-02-01' AND date'1993-02-10' + |""".stripMargin) + sourceDF1.write + .format("delta") + .mode(SaveMode.Overwrite) + .partitionBy("l_shipdate") + .save(dataPath) + + val result = spark.read + .format("delta") + .load(dataPath) + .count() + assert(result == 600572) + } + } + + test("test path based parquet update with the delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_update" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + + spark.sql(s""" + | update delta.`$dataPath` set l_returnflag = 'Z' where l_orderkey = 12647 + |""".stripMargin) + + { + val df = spark.read + .format("delta") + .load(dataPath) + .where("l_returnflag = 'Z'") + assert(df.count() == 1) + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val parquetScan = scanExec.head + assert(parquetScan.nodeName.startsWith("Scan parquet")) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 4) + } + + val clickhouseTable = DeltaTable.forPath(spark, dataPath) + clickhouseTable.updateExpr("l_orderkey = 10086", Map("l_returnflag" -> "'X'")) + + { + val df = spark.read + .format("delta") + .load(dataPath) + .where("l_returnflag = 'X'") + assert(df.count() == 1) + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val parquetScan = scanExec.head + assert(parquetScan.nodeName.startsWith("Scan parquet")) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 3) + } + + val df = spark.read + .format("delta") + .load(dataPath) + assert(df.count() == 600572) + } + + test("test path based parquet delete with the delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_delete" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + + spark.sql(s""" + | delete from delta.`$dataPath` where l_orderkey = 12647 + |""".stripMargin) + val df = spark.read + .format("delta") + .load(dataPath) + assert(df.count() == 600571) + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + val parquetScan = scanExec.head + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + assert(addFiles.size == 4) + + val clickhouseTable = DeltaTable.forPath(spark, dataPath) + clickhouseTable.delete("mod(l_orderkey, 3) = 2") + val df1 = spark.read + .format("delta") + .load(dataPath) + assert(df1.count() == 400089) + } + + test("test path based parquet upsert with the delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_upsert" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + + val df0 = spark.sql(s""" + | select count(*) from delta.`$dataPath` + |""".stripMargin) + assert( + df0.collect().apply(0).get(0) == 600572 + ) + upsertPathBasedSourceTableAndCheck(dataPath) + } + + private def upsertPathBasedSourceTableAndCheck(dataPath: String) = { + // Why selecting l_orderkey having count(*) =1 ? + // Answer: to avoid "org.apache.spark.sql.delta.DeltaUnsupportedOperationException: + // Cannot perform Merge as multiple source rows matched and attempted to modify the same + // target row in the Delta table in possibly conflicting ways." + spark.sql(s""" + merge into delta.`$dataPath` + using ( + + select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, + 'Z' as `l_returnflag`, + l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + from lineitem where l_orderkey in (select l_orderkey from lineitem group by l_orderkey having count(*) =1 ) and l_orderkey < 100000 + + union + + select l_orderkey + 10000000, + l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, + l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + from lineitem where l_orderkey in (select l_orderkey from lineitem group by l_orderkey having count(*) =1 ) and l_orderkey < 100000 + + ) as updates + on updates.l_orderkey = delta.`$dataPath`.l_orderkey + when matched then update set * + when not matched then insert * + """.stripMargin) + + { + val df1 = spark.sql(s""" + | select count(*) from delta.`$dataPath` + |""".stripMargin) + assert( + df1.collect().apply(0).get(0) == 600572 + 3506 + ) + } + { + val df2 = + spark.sql(s""" + | select count(*) from delta.`$dataPath` where l_returnflag = 'Z' + |""".stripMargin) + assert( + df2.collect().apply(0).get(0) == 3506 + ) + } + + { + val df3 = + spark.sql(s""" + | select count(*) from delta.`$dataPath` where l_orderkey > 10000000 + |""".stripMargin) + assert( + df3.collect().apply(0).get(0) == 3506 + ) + } + } + + test("test path based parquet write with partition + delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_partition" + clearDataPath(dataPath) + + val sourceDF = spark.sql(s""" + |select * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .partitionBy("l_shipdate", "l_returnflag") + .mode(SaveMode.Append) + .save(dataPath) + + val sourceDF1 = spark.sql( + s""" + |select * from lineitem where l_shipdate BETWEEN date'1993-01-01' AND date'1993-01-10' + |""".stripMargin) + + sourceDF1.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | delta.`$dataPath` + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr, compareResult = false) { + df => + val result = df.collect() + assert(result.size == 4) + assert(result(0).getString(0).equals("A")) + assert(result(0).getString(1).equals("F")) + assert(result(0).getDouble(2) == 3803858.0) + + assert(result(2).getString(0).equals("N")) + assert(result(2).getString(1).equals("O")) + assert(result(2).getDouble(2) == 7454519.0) + + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val parquetScan = scanExec(0) + assert(parquetScan.nodeName.startsWith("Scan parquet")) + assert(parquetScan.metrics("numFiles").value == 3744) + + val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] + val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) + + assert(addFiles.size == 3835) + assert( + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 2) + assert( + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) + assert( + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 2) + } + } + + test("test path based parquet CTAS with delta") { + val dataPath = s"$basePath/lineitem_delta_parquet_ctas" + clearDataPath(dataPath) + + spark.sql(s""" + |CREATE TABLE delta.`$dataPath` + |USING delta + |PARTITIONED BY (l_shipdate) + | as select * from lineitem + |""".stripMargin) + + val sqlStr = + s""" + |SELECT + | l_returnflag, + | l_linestatus, + | sum(l_quantity) AS sum_qty, + | sum(l_extendedprice) AS sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + | avg(l_quantity) AS avg_qty, + | avg(l_extendedprice) AS avg_price, + | avg(l_discount) AS avg_disc, + | count(*) AS count_order + |FROM + | delta.`$dataPath` + |WHERE + | l_shipdate <= date'1998-09-02' - interval 1 day + |GROUP BY + | l_returnflag, + | l_linestatus + |ORDER BY + | l_returnflag, + | l_linestatus; + | + |""".stripMargin + runTPCHQueryBySQL(1, sqlStr) { _ => {} } + } + + test("test parquet optimize basic") { + withSQLConf("spark.databricks.delta.optimize.maxFileSize" -> "20000000") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_optimize; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_optimize + |USING delta + |LOCATION '$basePath/lineitem_delta_parquet_optimize' + | as select /*+ REPARTITION(20) */ * from lineitem + |""".stripMargin) + + spark.sql("optimize lineitem_delta_parquet_optimize") + val ret = spark.sql("select count(*) from lineitem_delta_parquet_optimize").collect() + assert(ret.apply(0).get(0) == 600572) + + assert( + countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize")) == 24 + ) + } + } + + def countFiles(directory: File): Int = { + if (directory.exists && directory.isDirectory) { + val files = directory.listFiles + val count = files + .filter(!_.getName.endsWith(".crc")) + .count(_.isFile) + files.filter(_.isDirectory).map(countFiles).sum + count + } else { + 0 + } + } + + test("test parquet optimize partitioned by one low card column") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_optimize_p2; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_optimize_p2 + |USING delta + |PARTITIONED BY (l_returnflag) + |LOCATION '$basePath/lineitem_delta_parquet_optimize_p2' + | as select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + + spark.sparkContext.setJobGroup("test3", "test3") + spark.sql("optimize lineitem_delta_parquet_optimize_p2") + val job_ids = spark.sparkContext.statusTracker.getJobIdsForGroup("test3") + if (sparkVersion.equals("3.2")) { + assert(job_ids.size == 7) // WILL trigger actual merge job + } else { + assert(job_ids.size == 8) // WILL trigger actual merge job + } + + spark.sparkContext.clearJobGroup() + + val ret = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p2").collect() + assert(ret.apply(0).get(0) == 600572) + + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") + assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p2")) == 23) + spark.sql("VACUUM lineitem_delta_parquet_optimize_p2 RETAIN 0 HOURS") + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p2")) == 5) + } else { + assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p2")) == 7) + } + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") + + val ret2 = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p2").collect() + assert(ret2.apply(0).get(0) == 600572) + } + + test("test parquet optimize parallel delete") { + withSQLConf("spark.databricks.delta.vacuum.parallelDelete.enabled" -> "true") { + spark.sql(s""" + |DROP TABLE IF EXISTS lineitem_delta_parquet_optimize_p4; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_optimize_p4 + |USING delta + |PARTITIONED BY (l_linenumber,l_returnflag) + |LOCATION '$basePath/lineitem_delta_parquet_optimize_p4' + | as select /*+ REPARTITION(6) */ * from lineitem + |""".stripMargin) + + spark.sql("optimize lineitem_delta_parquet_optimize_p4") + val ret = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p4").collect() + assert(ret.apply(0).get(0) == 600572) + + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") + assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p4")) == 149) + spark.sql("VACUUM lineitem_delta_parquet_optimize_p4 RETAIN 0 HOURS") + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p4")) == 23) + } else { + assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p4")) == 25) + } + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") + + val ret2 = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p4").collect() + assert(ret2.apply(0).get(0) == 600572) + } + } + + test("test parquet optimize with the path based table") { + val dataPath = s"$basePath/lineitem_delta_parquet_optimize_path_based" + clearDataPath(dataPath) + withSQLConf( + ("spark.databricks.delta.optimize.maxFileSize" -> "1000000"), + ("spark.databricks.delta.optimize.minFileSize" -> "838000")) { + + val sourceDF = spark.sql(s""" + |select /*+ REPARTITION(50) */ * from lineitem + |""".stripMargin) + + sourceDF.write + .format("delta") + .mode(SaveMode.Append) + .save(dataPath) + + val clickhouseTable = DeltaTable.forPath(spark, dataPath) + clickhouseTable.optimize().executeCompaction() + + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") + clickhouseTable.vacuum(0.0) + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(dataPath)) == 27) + } else { + assert(countFiles(new File(dataPath)) == 29) + } + + val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() + assert(ret.apply(0).get(0) == 600572) + } + + withSQLConf( + ("spark.databricks.delta.optimize.maxFileSize" -> "10000000"), + ("spark.databricks.delta.optimize.minFileSize" -> "1000000")) { + + val clickhouseTable = DeltaTable.forPath(spark, dataPath) + clickhouseTable.optimize().executeCompaction() + + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") + clickhouseTable.vacuum(0.0) + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(dataPath)) == 6) + } else { + assert(countFiles(new File(dataPath)) == 12) + } + + val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() + assert(ret.apply(0).get(0) == 600572) + } + + // now merge all parts (testing merging from merged parts) + val clickhouseTable = DeltaTable.forPath(spark, dataPath) + clickhouseTable.optimize().executeCompaction() + + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") + clickhouseTable.vacuum(0.0) + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(dataPath)) == 5) + } else { + assert(countFiles(new File(dataPath)) == 13) + } + + val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() + assert(ret.apply(0).get(0) == 600572) + } +} +// scalastyle:off line.size.limit diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index f016f9dc5d14..c94a3bf50c63 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -127,7 +127,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) spark.sql("VACUUM lineitem_mergetree_optimize_p RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) + } else { + // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22730) + } + spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() @@ -150,7 +156,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sparkContext.setJobGroup("test2", "test2") spark.sql("optimize lineitem_mergetree_optimize_p2") val job_ids = spark.sparkContext.statusTracker.getJobIdsForGroup("test2") - assert(job_ids.size == 7) // WILL trigger actual merge job + if (sparkVersion.equals("3.2")) { + assert(job_ids.size == 7) // WILL trigger actual merge job + } else { + assert(job_ids.size == 8) // WILL trigger actual merge job + } + spark.sparkContext.clearJobGroup() val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() @@ -159,10 +170,18 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 372) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 239) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 239) + } else { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 241) + } spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") // the second VACUUM will remove some empty folders - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 220) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 220) + } else { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 226) + } spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() @@ -190,9 +209,17 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 306) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 306) + } else { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 308) + } spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 276) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 276) + } else { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 282) + } spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() @@ -221,9 +248,17 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 306) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 306) + } else { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 308) + } spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 276) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 276) + } else { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 282) + } spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() @@ -251,7 +286,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 99) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 99) + } else { + // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. + // this case will create a checkpoint + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 105) + } spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() @@ -271,7 +312,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) + } else { + // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 104) + } spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() @@ -284,7 +330,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 77) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 77) + } else { + // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) + } spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() @@ -318,7 +369,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 315 else 321 + if (sparkVersion.equals("3.2")) 315 else 327 }) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") @@ -393,7 +444,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") - assert(countFiles(new File(dataPath)) == 99) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(dataPath)) == 99) + } else { + assert(countFiles(new File(dataPath)) == 105) + } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() assert(ret.apply(0).get(0) == 600572) @@ -414,7 +469,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") - assert(countFiles(new File(dataPath)) == 93) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(dataPath)) == 93) + } else { + assert(countFiles(new File(dataPath)) == 104) + } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() assert(ret.apply(0).get(0) == 600572) @@ -428,7 +487,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") - assert(countFiles(new File(dataPath)) == 77) + if (sparkVersion.equals("3.2")) { + assert(countFiles(new File(dataPath)) == 77) + } else { + assert(countFiles(new File(dataPath)) == 93) + } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() assert(ret.apply(0).get(0) == 600572) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala index ff905251b8ae..f4855840256b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala @@ -177,6 +177,7 @@ abstract class FileSourceScanExecTransformerBase( relation.fileFormat.getClass.getSimpleName match { case "OrcFileFormat" => ReadFileFormat.OrcReadFormat case "ParquetFileFormat" => ReadFileFormat.ParquetReadFormat + case "DeltaParquetFileFormat" => ReadFileFormat.ParquetReadFormat case "DwrfFileFormat" => ReadFileFormat.DwrfReadFormat case "DeltaMergeTreeFileFormat" => ReadFileFormat.MergeTreeReadFormat case "CSVFileFormat" => ReadFileFormat.TextReadFormat diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml index 2396087fcc33..79afa94c8e32 100644 --- a/gluten-ut/pom.xml +++ b/gluten-ut/pom.xml @@ -92,7 +92,7 @@ io.delta ${delta.package.name}_${scala.binary.version} - provided + test diff --git a/pom.xml b/pom.xml index 0f37bcbf1851..63c53f109302 100644 --- a/pom.xml +++ b/pom.xml @@ -42,8 +42,6 @@ ${java.version} ${java.version} 2.9.3 - 2.0.1 - 20 2.12 2.12.15 3 @@ -155,8 +153,8 @@ 1.3.1 delta-core - 2.2.0 - 22 + 2.3.0 + 23 @@ -181,8 +179,8 @@ 3.5.1 1.5.0 delta-spark - 3.1.0 - 31 + 3.2.0 + 32 2.15.1 3.3.4 From 91f54072c00c4ab79db6de67d865ec3d2604d2a2 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 31 May 2024 20:45:35 +0800 Subject: [PATCH 191/402] [VL] Daily Update Velox Version (2024_05_31) (#5931) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index ff31daddb7b4..f02cab1ebfcb 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_30 +VELOX_BRANCH=2024_05_31 VELOX_HOME="" #Set on run gluten on HDFS From 26ff58d3b85485ca0a7b9d6bab2d82ae5aeff49b Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Sat, 1 Jun 2024 14:47:55 +0800 Subject: [PATCH 192/402] [GLUTEN-5944][CH] Fallback to run delta vacuum command (#5945) Fallback to run delta vacuum command: When AQE is on, now gluten CH backend + Delta ran delta vacuum command failed, fallback to run it first. Close #5944. --- .../sql/delta/commands/VacuumCommand.scala | 19 +++++++++++++ .../sql/delta/commands/VacuumCommand.scala | 19 +++++++++++++ ...utenClickHouseDeltaParquetWriteSuite.scala | 12 --------- ...utenClickHouseMergeTreeOptimizeSuite.scala | 27 ------------------- .../GlutenClickHouseTableAfterRestart.scala | 4 --- 5 files changed, 38 insertions(+), 43 deletions(-) diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 3a390f64d559..c5527933b2fc 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.delta.commands +import org.apache.gluten.utils.QueryPlanSelector + import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} @@ -141,6 +143,13 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { val relativizeIgnoreError = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) + // --- modified start + val originalEnabledGluten = + spark.sparkContext.getLocalProperty(QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY) + // gluten can not support vacuum command + spark.sparkContext.setLocalProperty(QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, "false") + // --- modified end + val validFiles = snapshot.stateDS .mapPartitions { actions => @@ -358,6 +367,16 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { spark.createDataset(Seq(basePath)).toDF("path") } finally { allFilesAndDirs.unpersist() + + // --- modified start + if (originalEnabledGluten != null) { + spark.sparkContext.setLocalProperty( + QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, originalEnabledGluten) + } else { + spark.sparkContext.setLocalProperty( + QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, "true") + } + // --- modified end } } } diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 5be548caf01c..9f82feeee2fc 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -33,6 +33,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.gluten.utils.QueryPlanSelector import org.apache.spark.broadcast.Broadcast import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig @@ -157,6 +158,14 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { val relativizeIgnoreError = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) val startTimeToIdentifyEligibleFiles = System.currentTimeMillis() + + // --- modified start + val originalEnabledGluten = + spark.sparkContext.getLocalProperty(QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY) + // gluten can not support vacuum command + spark.sparkContext.setLocalProperty(QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, "false") + // --- modified end + val validFiles = snapshot.stateDS .mapPartitions { actions => val reservoirBase = new Path(basePath) @@ -349,6 +358,16 @@ object VacuumCommand extends VacuumCommandImpl with Serializable { spark.createDataset(Seq(basePath)).toDF("path") } finally { allFilesAndDirs.unpersist() + + // --- modified start + if (originalEnabledGluten != null) { + spark.sparkContext.setLocalProperty( + QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, originalEnabledGluten) + } else { + spark.sparkContext.setLocalProperty( + QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, "true") + } + // --- modified end } } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala index a097fc6cd4ab..8fab604dee3c 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig - import org.apache.spark.SparkConf import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.actions.AddFile @@ -1311,7 +1309,6 @@ class GlutenClickHouseDeltaParquetWriteSuite val ret = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p2").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p2")) == 23) spark.sql("VACUUM lineitem_delta_parquet_optimize_p2 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -1319,7 +1316,6 @@ class GlutenClickHouseDeltaParquetWriteSuite } else { assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p2")) == 7) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p2").collect() assert(ret2.apply(0).get(0) == 600572) @@ -1343,7 +1339,6 @@ class GlutenClickHouseDeltaParquetWriteSuite val ret = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p4").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p4")) == 149) spark.sql("VACUUM lineitem_delta_parquet_optimize_p4 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -1351,7 +1346,6 @@ class GlutenClickHouseDeltaParquetWriteSuite } else { assert(countFiles(new File(s"$basePath/lineitem_delta_parquet_optimize_p4")) == 25) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_delta_parquet_optimize_p4").collect() assert(ret2.apply(0).get(0) == 600572) @@ -1377,9 +1371,7 @@ class GlutenClickHouseDeltaParquetWriteSuite val clickhouseTable = DeltaTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") if (sparkVersion.equals("3.2")) { assert(countFiles(new File(dataPath)) == 27) } else { @@ -1397,9 +1389,7 @@ class GlutenClickHouseDeltaParquetWriteSuite val clickhouseTable = DeltaTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") if (sparkVersion.equals("3.2")) { assert(countFiles(new File(dataPath)) == 6) } else { @@ -1414,9 +1404,7 @@ class GlutenClickHouseDeltaParquetWriteSuite val clickhouseTable = DeltaTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") if (sparkVersion.equals("3.2")) { assert(countFiles(new File(dataPath)) == 5) } else { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index c94a3bf50c63..650bbcc7b32c 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig - import org.apache.spark.SparkConf import org.apache.spark.sql.SaveMode import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper @@ -124,7 +122,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) spark.sql("VACUUM lineitem_mergetree_optimize_p RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -134,8 +131,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22730) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") - val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() assert(ret2.apply(0).get(0) == 600572) } @@ -167,7 +162,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 372) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -182,7 +176,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite } else { assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 226) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() assert(ret2.apply(0).get(0) == 600572) @@ -206,7 +199,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -220,7 +212,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite } else { assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 282) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() assert(ret2.apply(0).get(0) == 600572) @@ -245,7 +236,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 516) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -259,7 +249,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite } else { assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 282) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() assert(ret2.apply(0).get(0) == 600572) @@ -283,7 +272,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p5") - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -293,7 +281,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite // this case will create a checkpoint assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 105) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() assert(ret.apply(0).get(0) == 600572) @@ -309,7 +296,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p5") - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -318,7 +304,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 104) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() assert(ret.apply(0).get(0) == 600572) @@ -327,7 +312,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite // now merge all parts (testing merging from merged parts) spark.sql("optimize lineitem_mergetree_optimize_p5") - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { @@ -336,7 +320,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) } - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() assert(ret.apply(0).get(0) == 600572) @@ -362,7 +345,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p6").collect() assert(ret.apply(0).get(0) == 600572) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { if (sparkVersion.equals("3.2")) 499 else 528 }) @@ -371,7 +353,6 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { if (sparkVersion.equals("3.2")) 315 else 327 }) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p6").collect() assert(ret2.apply(0).get(0) == 600572) @@ -394,9 +375,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite |""".stripMargin) spark.sql("optimize lineitem_mergetree_index") - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("vacuum lineitem_mergetree_index") - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") val df = spark .sql(s""" @@ -440,10 +419,8 @@ class GlutenClickHouseMergeTreeOptimizeSuite val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") if (sparkVersion.equals("3.2")) { assert(countFiles(new File(dataPath)) == 99) } else { @@ -465,10 +442,8 @@ class GlutenClickHouseMergeTreeOptimizeSuite val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") if (sparkVersion.equals("3.2")) { assert(countFiles(new File(dataPath)) == 93) } else { @@ -483,10 +458,8 @@ class GlutenClickHouseMergeTreeOptimizeSuite val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.optimize().executeCompaction() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") if (sparkVersion.equals("3.2")) { assert(countFiles(new File(dataPath)) == 77) } else { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala index 9e55df0fa836..baf79436cf8b 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig - import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.SparkSession.{getActiveSession, getDefaultSession} @@ -250,9 +248,7 @@ class GlutenClickHouseTableAfterRestart restartSpark() - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false") spark.sql("vacuum table_restart_vacuum") - spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true") assert(spark.sql("select count(*) from table_restart_vacuum").collect().apply(0).get(0) == 4) } From 7a036dd6e1e386f49650aebece12860996eda12d Mon Sep 17 00:00:00 2001 From: Shuai li Date: Mon, 3 Jun 2024 09:57:36 +0800 Subject: [PATCH 193/402] [GLUTEN-5939][CH] Support java timezone id named 'GMT+8' or 'GMT+08:00' (#5940) * support java timezone * update ch --- .../GlutenClickHouseDatetimeExpressionSuite.scala | 3 ++- cpp-ch/clickhouse.version | 2 +- cpp-ch/local-engine/Common/CHUtil.cpp | 7 ++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala index 295ad4f25136..53416607521e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala @@ -41,6 +41,7 @@ class GlutenClickHouseDatetimeExpressionSuite .set("spark.sql.shuffle.partitions", "5") .set("spark.sql.autoBroadcastJoinThreshold", "10MB") .set("spark.sql.adaptive.enabled", "true") + .set("spark.sql.session.timeZone", "GMT+08:00") } override protected def createTPCHNotNullTables(): Unit = { @@ -143,7 +144,7 @@ class GlutenClickHouseDatetimeExpressionSuite | date_trunc('month', t) c |from date_trunc_tmp1 |""".stripMargin - compareResultsAgainstVanillaSpark(sql2, true, { _ => }, false) + compareResultsAgainstVanillaSpark(sql2, true, { _ => }) } } diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 5125aabe536d..97e9df5a7688 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence CH_BRANCH=rebase_ch/20240527 -CH_COMMIT=55b10ba376274f2a61a4c1daf1a2fb744155bd32 +CH_COMMIT=7ebb938593259aeb09952289ef7553b045ce4c15 diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 42787172af53..e37114756f34 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -585,12 +585,13 @@ void BackendInitializerUtil::initEnvs(DB::Context::ConfigurationPtr config) /// Set environment variable TZ if possible if (config->has("timezone")) { - const String timezone_name = config->getString("timezone"); - if (0 != setenv("TZ", timezone_name.data(), 1)) /// NOLINT + const std::string config_timezone = config->getString("timezone"); + const String mapped_timezone = DateLUT::mappingForJavaTimezone(config_timezone); + if (0 != setenv("TZ", mapped_timezone.data(), 1)) // NOLINT(concurrency-mt-unsafe) // ok if not called concurrently with other setenv/getenv throw Poco::Exception("Cannot setenv TZ variable"); tzset(); - DateLUT::setDefaultTimezone(timezone_name); + DateLUT::setDefaultTimezone(mapped_timezone); } /// Set environment variable LIBHDFS3_CONF if possible From ad817ed51f381d037a9de250e639bb6514e94242 Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Mon, 3 Jun 2024 09:59:33 +0800 Subject: [PATCH 194/402] [GLUTEN-5414] [VL] Support arrow csv option and schema (#5850) Support basic option now, will support more options after arrow patch merged. apache/arrow#41646 Before this patch, if the required schema is different with file schema, csv read will fallback. And changed to use index in file instead of check the file column name considering case sensitive. Add a new common test function when the rule applies to Logical plan. Compile arrow with version 15.0.0-gluten, upgrade arrow-dataset and arrow-c-data version from 15.0.0 to 15.0.0-gluten. --- .github/workflows/velox_docker.yml | 71 +- .github/workflows/velox_docker_cache.yml | 8 +- .../datasource/ArrowCSVFileFormat.scala | 111 ++- .../datasource/ArrowCSVOptionConverter.scala | 62 ++ .../datasource/ArrowConvertorRule.scala | 12 +- .../v2/ArrowCSVPartitionReaderFactory.scala | 79 +- .../gluten/datasource/v2/ArrowCSVTable.scala | 12 + .../extension/ArrowScanReplaceRule.scala | 3 +- .../datasource/csv/student_option.csv | 4 + .../datasource/csv/student_option_schema.csv | 4 + .../datasource/csv/student_option_str.csv | 4 + .../gluten/execution/TestOperator.scala | 235 ++++- ep/build-velox/src/build_velox.sh | 10 +- ep/build-velox/src/get_velox.sh | 2 + .../modify_arrow_dataset_scan_option.patch | 883 ++++++++++++++++++ ep/build-velox/src/modify_velox.patch | 3 +- .../WholeStageTransformerSuite.scala | 20 + gluten-data/pom.xml | 4 +- .../org/apache/gluten/utils/ArrowUtil.scala | 147 +-- gluten-ut/spark32/pom.xml | 2 +- .../utils/velox/VeloxTestSettings.scala | 9 + gluten-ut/spark33/pom.xml | 2 +- .../utils/velox/VeloxTestSettings.scala | 9 + gluten-ut/spark34/pom.xml | 2 +- .../utils/velox/VeloxTestSettings.scala | 9 + gluten-ut/spark35/pom.xml | 2 +- .../utils/velox/VeloxTestSettings.scala | 11 + .../datasources/csv/GlutenCSVSuite.scala | 65 +- gluten-ut/test/pom.xml | 2 +- pom.xml | 1 + 30 files changed, 1475 insertions(+), 313 deletions(-) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVOptionConverter.scala create mode 100644 backends-velox/src/test/resources/datasource/csv/student_option.csv create mode 100644 backends-velox/src/test/resources/datasource/csv/student_option_schema.csv create mode 100644 backends-velox/src/test/resources/datasource/csv/student_option_str.csv create mode 100644 ep/build-velox/src/modify_arrow_dataset_scan_option.patch diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index d11d4032ffde..bbd713d99077 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -59,16 +59,24 @@ jobs: id: cache uses: actions/cache/restore@v3 with: - path: ./cpp/build/releases/ + path: | + ./cpp/build/releases/ + ~/.m2/repository/org/apache/arrow/ key: cache-velox-build-${{ hashFiles('./cache-key') }} - name: Build Gluten Velox third party if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | source dev/ci-velox-buildstatic.sh - - uses: actions/upload-artifact@v2 + - name: Upload Artifact Native + uses: actions/upload-artifact@v2 with: path: ./cpp/build/releases/ name: velox-native-lib-centos-7-${{github.sha}} + - name: Upload Artifact Arrow Jar + uses: actions/upload-artifact@v2 + with: + path: /root/.m2/repository/org/apache/arrow/ + name: velox-arrow-jar-centos-7-${{github.sha}} run-tpc-test-ubuntu: needs: build-native-lib-centos-7 @@ -92,11 +100,16 @@ jobs: container: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: Download All Artifacts + - name: Download All Native Artifacts uses: actions/download-artifact@v2 with: name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | if [ "${{ matrix.java }}" = "java-17" ]; then @@ -105,6 +118,7 @@ jobs: apt-get update && apt-get install -y openjdk-8-jdk maven fi apt remove openjdk-11* -y + ls -l /root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/ - name: Build and run TPCH/DS run: | cd $GITHUB_WORKSPACE/ @@ -140,11 +154,16 @@ jobs: container: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: Download All Artifacts + - name: Download All Native Artifacts uses: actions/download-artifact@v2 with: name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list if: matrix.os == 'centos:8' run: | @@ -215,11 +234,16 @@ jobs: sudo docker image prune --all --force > /dev/null df -h - uses: actions/checkout@v2 - - name: Download All Artifacts + - name: Download All Native Artifacts uses: actions/download-artifact@v2 with: name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | sudo apt-get update @@ -312,11 +336,16 @@ jobs: sudo docker image prune --all --force > /dev/null df -h - uses: actions/checkout@v2 - - name: Download All Artifacts + - name: Download All Native Artifacts uses: actions/download-artifact@v2 with: name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | sudo apt-get update @@ -400,11 +429,16 @@ jobs: container: centos:8 steps: - uses: actions/checkout@v2 - - name: Download All Artifacts + - name: Download All Native Artifacts uses: actions/download-artifact@v2 with: name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true @@ -461,11 +495,16 @@ jobs: container: ubuntu:22.04 steps: - uses: actions/checkout@v2 - - name: Download All Artifacts + - name: Download All Native Artifacts uses: actions/download-artifact@v2 with: name: velox-native-lib-centos-7-${{github.sha}} - path: ./cpp/build/releases + path: ./cpp/build/releases/ + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget @@ -506,7 +545,9 @@ jobs: id: cache uses: actions/cache/restore@v3 with: - path: ./cpp/build/releases/ + path: | + ./cpp/build/releases/ + /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }} - name: Setup build dependency if: ${{ steps.cache.outputs.cache-hit != 'true' }} diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 44271c4fc0d0..3c05acf7eca2 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -38,7 +38,9 @@ jobs: uses: actions/cache/restore@v3 with: lookup-only: true - path: ./cpp/build/releases/ + path: | + ./cpp/build/releases/ + /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-${{ hashFiles('./cache-key') }} - name: Build Gluten Velox third party if: steps.check-cache.outputs.cache-hit != 'true' @@ -49,7 +51,9 @@ jobs: id: cache uses: actions/cache/save@v3 with: - path: ./cpp/build/releases/ + path: | + ./cpp/build/releases/ + /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-${{ hashFiles('./cache-key') }} # ccache-native-lib-ubuntu-velox-ut: diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala index 0f6813d8fc6a..7c3ca8fc8cde 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala @@ -40,8 +40,10 @@ import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.SerializableConfiguration +import org.apache.arrow.c.ArrowSchema import org.apache.arrow.dataset.file.FileSystemDatasetFactory import org.apache.arrow.dataset.scanner.ScanOptions +import org.apache.arrow.dataset.scanner.csv.CsvFragmentScanOptions import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.VectorUnloader import org.apache.arrow.vector.types.pojo.Schema @@ -51,11 +53,17 @@ import org.apache.hadoop.fs.{FileStatus, Path} import java.net.URLDecoder import java.util.Optional -import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.JavaConverters.{asJavaIterableConverter, asScalaBufferConverter} -class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging with Serializable { +class ArrowCSVFileFormat(parsedOptions: CSVOptions) + extends FileFormat + with DataSourceRegister + with Logging + with Serializable { private val fileFormat = org.apache.arrow.dataset.file.FileFormat.CSV + private lazy val pool = ArrowNativeMemoryPool.arrowPool("FileSystem Read") + var fallback = false override def isSplitable( sparkSession: SparkSession, @@ -68,9 +76,11 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging sparkSession: SparkSession, options: Map[String, String], files: Seq[FileStatus]): Option[StructType] = { + val arrowConfig = ArrowCSVOptionConverter.convert(parsedOptions) ArrowUtil.readSchema( files, fileFormat, + arrowConfig, ArrowBufferAllocators.contextInstance(), ArrowNativeMemoryPool.arrowPool("infer schema")) } @@ -89,51 +99,74 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging val broadcastedHadoopConf = sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) val batchSize = sqlConf.columnBatchSize - val caseSensitive = sqlConf.caseSensitiveAnalysis val columnPruning = sqlConf.csvColumnPruning && !requiredSchema.exists(_.name == sparkSession.sessionState.conf.columnNameOfCorruptRecord) - val parsedOptions = new CSVOptions( - options, - columnPruning, - sparkSession.sessionState.conf.sessionLocalTimeZone, - sparkSession.sessionState.conf.columnNameOfCorruptRecord) val actualFilters = filters.filterNot(_.references.contains(parsedOptions.columnNameOfCorruptRecord)) (file: PartitionedFile) => { + val actualDataSchema = StructType( + dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) + val actualRequiredSchema = StructType( + requiredSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) ArrowCSVFileFormat.checkHeader( file, - dataSchema, - requiredSchema, + actualDataSchema, + actualRequiredSchema, parsedOptions, actualFilters, broadcastedHadoopConf.value.value) - val factory = - ArrowUtil.makeArrowDiscovery( + + val arrowConfig = ArrowCSVOptionConverter.convert(parsedOptions) + val allocator = ArrowBufferAllocators.contextInstance() + // todo predicate validation / pushdown + val fileNames = ArrowUtil + .readArrowFileColumnNames( URLDecoder.decode(file.filePath.toString, "UTF-8"), fileFormat, + arrowConfig, ArrowBufferAllocators.contextInstance(), - ArrowNativeMemoryPool.arrowPool("FileSystemDatasetFactory") - ) - // todo predicate validation / pushdown - val fileFields = factory.inspect().getFields.asScala + pool) + val tokenIndexArr = + actualRequiredSchema + .map(f => java.lang.Integer.valueOf(actualDataSchema.indexOf(f))) + .toArray + val fileIndex = tokenIndexArr.filter(_ < fileNames.length) + val requestSchema = new StructType( + fileIndex + .map(index => StructField(fileNames(index), actualDataSchema(index).dataType))) + val missingIndex = tokenIndexArr.filter(_ >= fileNames.length) + val missingSchema = new StructType(missingIndex.map(actualDataSchema(_))) // TODO: support array/map/struct types in out-of-order schema reading. + val cSchema: ArrowSchema = ArrowSchema.allocateNew(allocator) + val cSchema2: ArrowSchema = ArrowSchema.allocateNew(allocator) try { - val actualReadFields = - ArrowUtil.getRequestedField(requiredSchema, fileFields, caseSensitive) + ArrowCSVOptionConverter.schema(requestSchema, cSchema, allocator, arrowConfig) + val factory = + ArrowUtil.makeArrowDiscovery( + URLDecoder.decode(file.filePath.toString, "UTF-8"), + fileFormat, + Optional.of(arrowConfig), + ArrowBufferAllocators.contextInstance(), + pool) + val fields = factory.inspect().getFields + val actualReadFields = new Schema( + fileIndex.map(index => fields.get(index)).toIterable.asJava) + ArrowCSVOptionConverter.schema(requestSchema, cSchema2, allocator, arrowConfig) ArrowCSVFileFormat .readArrow( ArrowBufferAllocators.contextInstance(), file, actualReadFields, - caseSensitive, - requiredSchema, + missingSchema, partitionSchema, factory, - batchSize) + batchSize, + arrowConfig) .asInstanceOf[Iterator[InternalRow]] } catch { case e: SchemaMismatchException => logWarning(e.getMessage) + fallback = true val iter = ArrowCSVFileFormat.fallbackReadVanilla( dataSchema, requiredSchema, @@ -148,8 +181,10 @@ class ArrowCSVFileFormat extends FileFormat with DataSourceRegister with Logging .rowToColumn(schema, batchSize, rows) .asInstanceOf[Iterator[InternalRow]] case d: Exception => throw d + } finally { + cSchema.close() + cSchema2.close() } - } } @@ -184,28 +219,23 @@ object ArrowCSVFileFormat { allocator: BufferAllocator, file: PartitionedFile, actualReadFields: Schema, - caseSensitive: Boolean, - requiredSchema: StructType, + missingSchema: StructType, partitionSchema: StructType, factory: FileSystemDatasetFactory, - batchSize: Int): Iterator[ColumnarBatch] = { - val compare = ArrowUtil.compareStringFunc(caseSensitive) + batchSize: Int, + arrowConfig: CsvFragmentScanOptions): Iterator[ColumnarBatch] = { val actualReadFieldNames = actualReadFields.getFields.asScala.map(_.getName).toArray - val actualReadSchema = new StructType( - actualReadFieldNames.map(f => requiredSchema.find(field => compare(f, field.name)).get)) val dataset = factory.finish(actualReadFields) - - val hasMissingColumns = actualReadFields.getFields.size() != requiredSchema.size - - val scanOptions = new ScanOptions(batchSize, Optional.of(actualReadFieldNames)) + val scanOptions = new ScanOptions.Builder(batchSize) + .columns(Optional.of(actualReadFieldNames)) + .fragmentScanOptions(arrowConfig) + .build() val scanner = dataset.newScan(scanOptions) val partitionVectors = ArrowUtil.loadPartitionColumns(batchSize, partitionSchema, file.partitionValues) - val nullVectors = if (hasMissingColumns) { - val missingSchema = - new StructType(requiredSchema.filterNot(actualReadSchema.contains).toArray) + val nullVectors = if (missingSchema.nonEmpty) { ArrowUtil.loadMissingColumns(batchSize, missingSchema) } else { Array.empty[ArrowWritableColumnVector] @@ -225,8 +255,7 @@ object ArrowCSVFileFormat { val batch = ArrowUtil.loadBatch( allocator, unloader.getRecordBatch, - actualReadSchema, - requiredSchema, + actualReadFields, partitionVectors, nullVectors) batch @@ -246,8 +275,8 @@ object ArrowCSVFileFormat { def checkHeader( file: PartitionedFile, - dataSchema: StructType, - requiredSchema: StructType, + actualDataSchema: StructType, + actualRequiredSchema: StructType, parsedOptions: CSVOptions, actualFilters: Seq[Filter], conf: Configuration): Unit = { @@ -255,10 +284,6 @@ object ArrowCSVFileFormat { if (!isStartOfFile) { return } - val actualDataSchema = StructType( - dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) - val actualRequiredSchema = StructType( - requiredSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord)) val parser = new UnivocityParser(actualDataSchema, actualRequiredSchema, parsedOptions, actualFilters) val schema = if (parsedOptions.columnPruning) actualRequiredSchema else actualDataSchema diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVOptionConverter.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVOptionConverter.scala new file mode 100644 index 000000000000..7d6a54c2ac7a --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVOptionConverter.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.datasource + +import org.apache.gluten.utils.ArrowAbiUtil + +import org.apache.spark.sql.catalyst.csv.CSVOptions +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.utils.SparkSchemaUtil + +import com.google.common.collect.ImmutableMap +import org.apache.arrow.c.ArrowSchema +import org.apache.arrow.dataset.scanner.csv.{CsvConvertOptions, CsvFragmentScanOptions} +import org.apache.arrow.memory.BufferAllocator + +import java.util + +object ArrowCSVOptionConverter { + def convert(option: CSVOptions): CsvFragmentScanOptions = { + val parseMap = new util.HashMap[String, String]() + val default = new CSVOptions( + CaseInsensitiveMap(Map()), + option.columnPruning, + SparkSchemaUtil.getLocalTimezoneID) + parseMap.put("strings_can_be_null", "true") + if (option.delimiter != default.delimiter) { + parseMap.put("delimiter", option.delimiter) + } + if (option.escapeQuotes != default.escapeQuotes) { + parseMap.put("quoting", (!option.escapeQuotes).toString) + } + + val convertOptions = new CsvConvertOptions(ImmutableMap.of()) + new CsvFragmentScanOptions(convertOptions, ImmutableMap.of(), parseMap) + } + + def schema( + requiredSchema: StructType, + cSchema: ArrowSchema, + allocator: BufferAllocator, + option: CsvFragmentScanOptions): Unit = { + val schema = SparkSchemaUtil.toArrowSchema(requiredSchema) + ArrowAbiUtil.exportSchema(allocator, schema, cSchema) + option.getConvertOptions.setArrowSchema(cSchema) + } + +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala index dab1ffd3b9e3..2778710155bf 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowConvertorRule.scala @@ -19,6 +19,7 @@ package org.apache.gluten.datasource import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.datasource.v2.ArrowCSVTable import org.apache.gluten.sql.shims.SparkShimLoader +import org.apache.gluten.utils.LogicalPlanSelector import org.apache.spark.annotation.Experimental import org.apache.spark.sql.SparkSession @@ -39,7 +40,7 @@ import scala.collection.convert.ImplicitConversions.`map AsScala` @Experimental case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { - override def apply(plan: LogicalPlan): LogicalPlan = { + override def apply(plan: LogicalPlan): LogicalPlan = LogicalPlanSelector.maybe(session, plan) { if (!BackendsApiManager.getSettings.enableNativeArrowReadFiles()) { return plan } @@ -49,7 +50,11 @@ case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { _, _, _) if validate(session, dataSchema, options) => - l.copy(relation = r.copy(fileFormat = new ArrowCSVFileFormat())(session)) + val csvOptions = new CSVOptions( + options, + columnPruning = session.sessionState.conf.csvColumnPruning, + session.sessionState.conf.sessionLocalTimeZone) + l.copy(relation = r.copy(fileFormat = new ArrowCSVFileFormat(csvOptions))(session)) case d @ DataSourceV2Relation( t @ CSVTable( name, @@ -88,7 +93,8 @@ case class ArrowConvertorRule(session: SparkSession) extends Rule[LogicalPlan] { } private def checkCsvOptions(csvOptions: CSVOptions, timeZone: String): Boolean = { - csvOptions.headerFlag && !csvOptions.multiLine && csvOptions.delimiter == "," && + csvOptions.headerFlag && !csvOptions.multiLine && + csvOptions.delimiter.length == 1 && csvOptions.quote == '\"' && csvOptions.escape == '\\' && csvOptions.lineSeparator.isEmpty && diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala index ddc7f797fb93..4af5022a6252 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.datasource.v2 -import org.apache.gluten.datasource.ArrowCSVFileFormat +import org.apache.gluten.datasource.{ArrowCSVFileFormat, ArrowCSVOptionConverter} import org.apache.gluten.exception.SchemaMismatchException import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool @@ -31,15 +31,17 @@ import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.execution.datasources.v2.FilePartitionReaderFactory import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.Filter -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.{SerializableConfiguration, TaskResources} -import org.apache.arrow.dataset.file.FileFormat +import org.apache.arrow.c.ArrowSchema +import org.apache.arrow.vector.types.pojo.Schema import java.net.URLDecoder +import java.util.Optional -import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.JavaConverters.asJavaIterableConverter case class ArrowCSVPartitionReaderFactory( sqlConf: SQLConf, @@ -53,8 +55,9 @@ case class ArrowCSVPartitionReaderFactory( with Logging { private val batchSize = sqlConf.parquetVectorizedReaderBatchSize - private val caseSensitive: Boolean = sqlConf.caseSensitiveAnalysis private val csvColumnPruning: Boolean = sqlConf.csvColumnPruning + private val fileFormat = org.apache.arrow.dataset.file.FileFormat.CSV + var fallback = false override def supportColumnarReads(partition: InputPartition): Boolean = true @@ -67,12 +70,12 @@ case class ArrowCSVPartitionReaderFactory( partitionedFile: PartitionedFile): PartitionReader[ColumnarBatch] = { val actualDataSchema = StructType( dataSchema.filterNot(_.name == options.columnNameOfCorruptRecord)) - val actualReadDataSchema = StructType( + val actualRequiredSchema = StructType( readDataSchema.filterNot(_.name == options.columnNameOfCorruptRecord)) ArrowCSVFileFormat.checkHeader( partitionedFile, actualDataSchema, - actualReadDataSchema, + actualRequiredSchema, options, filters, broadcastedConf.value.value) @@ -87,29 +90,54 @@ case class ArrowCSVPartitionReaderFactory( ArrowBufferAllocators.contextInstance(), ArrowNativeMemoryPool.arrowPool("FileSystemFactory")) } - val factory = ArrowUtil.makeArrowDiscovery( - URLDecoder.decode(partitionedFile.filePath.toString(), "UTF-8"), - FileFormat.CSV, - allocator, - pool) - val parquetFileFields = factory.inspect().getFields.asScala + val arrowConfig = ArrowCSVOptionConverter.convert(options) + val fileNames = ArrowUtil + .readArrowFileColumnNames( + URLDecoder.decode(partitionedFile.filePath.toString, "UTF-8"), + fileFormat, + arrowConfig, + ArrowBufferAllocators.contextInstance(), + pool) + val tokenIndexArr = + actualRequiredSchema.map(f => java.lang.Integer.valueOf(actualDataSchema.indexOf(f))).toArray + val fileIndex = tokenIndexArr.filter(_ < fileNames.length) + val requestSchema = new StructType( + fileIndex + .map(index => StructField(fileNames(index), actualDataSchema(index).dataType))) + val missingIndex = tokenIndexArr.filter(_ >= fileNames.length) + val missingSchema = new StructType(missingIndex.map(actualDataSchema(_))) + // TODO: support array/map/struct types in out-of-order schema reading. + val cSchema: ArrowSchema = ArrowSchema.allocateNew(allocator) + val cSchema2: ArrowSchema = ArrowSchema.allocateNew(allocator) // TODO: support array/map/struct types in out-of-order schema reading. val iter = try { - val actualReadFields = - ArrowUtil.getRequestedField(readDataSchema, parquetFileFields, caseSensitive) - ArrowCSVFileFormat.readArrow( - allocator, - partitionedFile, - actualReadFields, - caseSensitive, - readDataSchema, - readPartitionSchema, - factory, - batchSize) + ArrowCSVOptionConverter.schema(requestSchema, cSchema, allocator, arrowConfig) + val factory = + ArrowUtil.makeArrowDiscovery( + URLDecoder.decode(partitionedFile.filePath.toString, "UTF-8"), + fileFormat, + Optional.of(arrowConfig), + ArrowBufferAllocators.contextInstance(), + pool) + val fields = factory.inspect().getFields + val actualReadFields = new Schema( + fileIndex.map(index => fields.get(index)).toIterable.asJava) + ArrowCSVOptionConverter.schema(requestSchema, cSchema2, allocator, arrowConfig) + ArrowCSVFileFormat + .readArrow( + ArrowBufferAllocators.contextInstance(), + partitionedFile, + actualReadFields, + missingSchema, + readPartitionSchema, + factory, + batchSize, + arrowConfig) } catch { case e: SchemaMismatchException => logWarning(e.getMessage) + fallback = true val iter = ArrowCSVFileFormat.fallbackReadVanilla( dataSchema, readDataSchema, @@ -125,6 +153,9 @@ case class ArrowCSVPartitionReaderFactory( partitionedFile) ArrowCSVFileFormat.rowToColumn(schema, batchSize, rows) case d: Exception => throw d + } finally { + cSchema.close() + cSchema2.close() } new PartitionReader[ColumnarBatch] { diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala index aa7f737f9cfc..02485975e705 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala @@ -16,11 +16,13 @@ */ package org.apache.gluten.datasource.v2 +import org.apache.gluten.datasource.ArrowCSVOptionConverter import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool import org.apache.gluten.utils.ArrowUtil import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} import org.apache.spark.sql.execution.datasources.FileFormat @@ -31,6 +33,8 @@ import org.apache.spark.util.TaskResources import org.apache.hadoop.fs.FileStatus +import scala.collection.JavaConverters.mapAsScalaMapConverter + case class ArrowCSVTable( name: String, sparkSession: SparkSession, @@ -48,9 +52,17 @@ case class ArrowCSVTable( } else { (ArrowBufferAllocators.contextInstance(), ArrowNativeMemoryPool.arrowPool("inferSchema")) } + val parsedOptions: CSVOptions = new CSVOptions( + options.asScala.toMap, + columnPruning = sparkSession.sessionState.conf.csvColumnPruning, + sparkSession.sessionState.conf.sessionLocalTimeZone, + sparkSession.sessionState.conf.columnNameOfCorruptRecord + ) + val arrowConfig = ArrowCSVOptionConverter.convert(parsedOptions) ArrowUtil.readSchema( files.head, org.apache.arrow.dataset.file.FileFormat.CSV, + arrowConfig, allocator, pool ) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala index adfc6ca742c9..dba8df5cf1a1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/ArrowScanReplaceRule.scala @@ -19,6 +19,7 @@ package org.apache.gluten.extension import org.apache.gluten.datasource.ArrowCSVFileFormat import org.apache.gluten.datasource.v2.ArrowCSVScan import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec +import org.apache.gluten.utils.PhysicalPlanSelector import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule @@ -26,7 +27,7 @@ import org.apache.spark.sql.execution.{ArrowFileSourceScanExec, FileSourceScanEx import org.apache.spark.sql.execution.datasources.v2.BatchScanExec case class ArrowScanReplaceRule(spark: SparkSession) extends Rule[SparkPlan] { - override def apply(plan: SparkPlan): SparkPlan = { + override def apply(plan: SparkPlan): SparkPlan = PhysicalPlanSelector.maybe(spark, plan) { plan.transformUp { case plan: FileSourceScanExec if plan.relation.fileFormat.isInstanceOf[ArrowCSVFileFormat] => ArrowFileSourceScanExec(plan) diff --git a/backends-velox/src/test/resources/datasource/csv/student_option.csv b/backends-velox/src/test/resources/datasource/csv/student_option.csv new file mode 100644 index 000000000000..919b7387b53c --- /dev/null +++ b/backends-velox/src/test/resources/datasource/csv/student_option.csv @@ -0,0 +1,4 @@ +Name;Language +Juno;Java +Peter;Python +Celin;C++ diff --git a/backends-velox/src/test/resources/datasource/csv/student_option_schema.csv b/backends-velox/src/test/resources/datasource/csv/student_option_schema.csv new file mode 100644 index 000000000000..be8459a21739 --- /dev/null +++ b/backends-velox/src/test/resources/datasource/csv/student_option_schema.csv @@ -0,0 +1,4 @@ +id,name,language +1,Juno,Java +2,Peter,Python +3,Celin,C++ diff --git a/backends-velox/src/test/resources/datasource/csv/student_option_str.csv b/backends-velox/src/test/resources/datasource/csv/student_option_str.csv new file mode 100644 index 000000000000..b4214b390cae --- /dev/null +++ b/backends-velox/src/test/resources/datasource/csv/student_option_str.csv @@ -0,0 +1,4 @@ +Name,Language +Juno,Java +Peter,Python +,C++ diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 7088b7b072d9..905d30055795 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DecimalType, StringType, StructField, StructType} +import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType, StructField, StructType} import java.util.concurrent.TimeUnit @@ -483,42 +483,153 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } test("csv scan") { - val filePath = rootPath + "/datasource/csv/student.csv" - val df = spark.read - .format("csv") - .option("header", "true") - .load(filePath) - df.createOrReplaceTempView("student") - runQueryAndCompare("select * from student") { - df => - val plan = df.queryExecution.executedPlan - assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) - assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) - val scan = plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).toList.head - assert( - scan - .asInstanceOf[ArrowFileSourceScanExec] - .relation - .fileFormat - .isInstanceOf[ArrowCSVFileFormat]) + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + } + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) + val scan = plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).toList.head + assert( + scan + .asInstanceOf[ArrowFileSourceScanExec] + .relation + .fileFormat + .isInstanceOf[ArrowCSVFileFormat]) + } + + test("csv scan with option string as null") { + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student_option_str.csv" + // test strings as null + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + } + val plan = df.queryExecution.executedPlan + assert(plan.find(_.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) + } + + test("csv scan with option delimiter") { + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student_option.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .option("delimiter", ";") + .load(filePath) + df.createOrReplaceTempView("student") + } + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) + } + + test("csv scan with schema") { + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student_option_schema.csv" + val schema = new StructType() + .add("id", StringType) + .add("name", StringType) + .add("language", StringType) + val df = spark.read + .schema(schema) + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + } + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + val scan = plan.find(_.isInstanceOf[ArrowFileSourceScanExec]) + assert(scan.isDefined) + assert( + !scan.get + .asInstanceOf[ArrowFileSourceScanExec] + .original + .relation + .fileFormat + .asInstanceOf[ArrowCSVFileFormat] + .fallback) + } + + test("csv scan with missing columns") { + val df = runAndCompare("select languagemissing, language, id_new_col from student") { + val filePath = rootPath + "/datasource/csv/student_option_schema.csv" + val schema = new StructType() + .add("id_new_col", IntegerType) + .add("name", StringType) + .add("language", StringType) + .add("languagemissing", StringType) + val df = spark.read + .schema(schema) + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") } + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[VeloxColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) + } + + test("csv scan with different name") { + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student_option_schema.csv" + val schema = new StructType() + .add("id_new_col", StringType) + .add("name", StringType) + .add("language", StringType) + val df = spark.read + .schema(schema) + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + } + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) + + val df2 = runAndCompare("select * from student_schema") { + val filePath = rootPath + "/datasource/csv/student_option_schema.csv" + val schema = new StructType() + .add("name", StringType) + .add("language", StringType) + val df = spark.read + .schema(schema) + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student_schema") + } + val plan2 = df2.queryExecution.executedPlan + assert(plan2.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan2.find(_.isInstanceOf[ArrowFileSourceScanExec]).isDefined) } test("csv scan with filter") { - val filePath = rootPath + "/datasource/csv/student.csv" - val df = spark.read - .format("csv") - .option("header", "true") - .load(filePath) - df.createOrReplaceTempView("student") - runQueryAndCompare("select * from student where Name = 'Peter'") { - df => - assert(df.queryExecution.executedPlan.find(s => s.isInstanceOf[ColumnarToRowExec]).isEmpty) - assert( - df.queryExecution.executedPlan - .find(s => s.isInstanceOf[ArrowFileSourceScanExec]) - .isDefined) + val df = runAndCompare("select * from student where Name = 'Peter'") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") } + assert(df.queryExecution.executedPlan.find(s => s.isInstanceOf[ColumnarToRowExec]).isEmpty) + assert( + df.queryExecution.executedPlan + .find(s => s.isInstanceOf[ArrowFileSourceScanExec]) + .isDefined) } test("insert into select from csv") { @@ -540,21 +651,55 @@ class TestOperator extends VeloxWholeStageTransformerSuite { test("csv scan datasource v2") { withSQLConf("spark.sql.sources.useV1SourceList" -> "") { - val filePath = rootPath + "/datasource/csv/student.csv" - val df = spark.read - .format("csv") - .option("header", "true") - .load(filePath) - df.createOrReplaceTempView("student") - runQueryAndCompare("select * from student") { - checkGlutenOperatorMatch[ArrowBatchScanExec] + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") } - runQueryAndCompare("select * from student where Name = 'Peter'") { - df => - val plan = df.queryExecution.executedPlan - assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isEmpty) - assert(plan.find(s => s.isInstanceOf[ArrowBatchScanExec]).isDefined) + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(s => s.isInstanceOf[ArrowBatchScanExec]).isDefined) + } + } + + test("csv scan datasource v2 with filter") { + withSQLConf("spark.sql.sources.useV1SourceList" -> "") { + val df = runAndCompare("select * from student where Name = 'Peter'") { + val filePath = rootPath + "/datasource/csv/student.csv" + val df = spark.read + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") + } + + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isEmpty) + assert(plan.find(s => s.isInstanceOf[ArrowBatchScanExec]).isDefined) + } + } + + test("csv scan with schema datasource v2") { + withSQLConf("spark.sql.sources.useV1SourceList" -> "") { + val df = runAndCompare("select * from student") { + val filePath = rootPath + "/datasource/csv/student_option_schema.csv" + val schema = new StructType() + .add("id", StringType) + .add("name", StringType) + .add("language", StringType) + val df = spark.read + .schema(schema) + .format("csv") + .option("header", "true") + .load(filePath) + df.createOrReplaceTempView("student") } + val plan = df.queryExecution.executedPlan + assert(plan.find(s => s.isInstanceOf[ColumnarToRowExec]).isDefined) + assert(plan.find(_.isInstanceOf[ArrowBatchScanExec]).isDefined) } } diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 3fc0dc6f661f..c13b49f30598 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -282,9 +282,11 @@ function compile_arrow_java_module() { ARROW_INSTALL_DIR="${ARROW_HOME}/../../install" pushd $ARROW_HOME/java - - mvn clean install -pl maven/module-info-compiler-maven-plugin -am \ - -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip + # Because arrow-bom module need the -DprocessAllModules + mvn versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules + + mvn clean install -am \ + -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip # Arrow C Data Interface CPP libraries mvn generate-resources -P generate-libs-cdata-all-os -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \ @@ -297,7 +299,7 @@ function compile_arrow_java_module() { -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N # Arrow Java libraries - mvn clean install -Parrow-jni -P arrow-c-data -pl dataset,c -am \ + mvn install -Parrow-jni -P arrow-c-data -pl c,dataset -am \ -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.cpp.build.dir=$ARROW_INSTALL_DIR/lib \ -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip popd diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index f02cab1ebfcb..ce25b20d43b8 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -256,8 +256,10 @@ function apply_compilation_fixes { velox_home=$2 sudo cp ${current_dir}/modify_velox.patch ${velox_home}/ sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/third_party/ + sudo cp ${current_dir}/modify_arrow_dataset_scan_option.patch ${velox_home}/third_party/ git add ${velox_home}/modify_velox.patch # to avoid the file from being deleted by git clean -dffx :/ git add ${velox_home}/third_party/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/ + git add ${velox_home}/third_party/modify_arrow_dataset_scan_option.patch # to avoid the file from being deleted by git clean -dffx :/ cd ${velox_home} echo "Applying patch to Velox source code..." git apply modify_velox.patch diff --git a/ep/build-velox/src/modify_arrow_dataset_scan_option.patch b/ep/build-velox/src/modify_arrow_dataset_scan_option.patch new file mode 100644 index 000000000000..4af78c030c00 --- /dev/null +++ b/ep/build-velox/src/modify_arrow_dataset_scan_option.patch @@ -0,0 +1,883 @@ +diff --git a/cpp/src/arrow/dataset/file_csv.cc b/cpp/src/arrow/dataset/file_csv.cc +index 09ab77572..f09377cf9 100644 +--- a/cpp/src/arrow/dataset/file_csv.cc ++++ b/cpp/src/arrow/dataset/file_csv.cc +@@ -24,6 +24,7 @@ + #include + #include + ++#include "arrow/c/bridge.h" + #include "arrow/csv/options.h" + #include "arrow/csv/parser.h" + #include "arrow/csv/reader.h" +@@ -52,6 +53,9 @@ using internal::Executor; + using internal::SerialExecutor; + + namespace dataset { ++namespace { ++inline bool parseBool(const std::string& value) { return value == "true" ? true : false; } ++} // namespace + + struct CsvInspectedFragment : public InspectedFragment { + CsvInspectedFragment(std::vector column_names, +@@ -503,5 +507,33 @@ Future<> CsvFileWriter::FinishInternal() { + return Status::OK(); + } + ++Result> CsvFragmentScanOptions::from( ++ const std::unordered_map& configs) { ++ std::shared_ptr options = ++ std::make_shared(); ++ for (auto const& it : configs) { ++ auto& key = it.first; ++ auto& value = it.second; ++ if (key == "delimiter") { ++ options->parse_options.delimiter = value.data()[0]; ++ } else if (key == "quoting") { ++ options->parse_options.quoting = parseBool(value); ++ } else if (key == "column_types") { ++ int64_t schema_address = std::stol(value); ++ ArrowSchema* cSchema = reinterpret_cast(schema_address); ++ ARROW_ASSIGN_OR_RAISE(auto schema, arrow::ImportSchema(cSchema)); ++ auto& column_types = options->convert_options.column_types; ++ for (auto field : schema->fields()) { ++ column_types[field->name()] = field->type(); ++ } ++ } else if (key == "strings_can_be_null") { ++ options->convert_options.strings_can_be_null = parseBool(value); ++ } else { ++ return Status::Invalid("Config " + it.first + "is not supported."); ++ } ++ } ++ return options; ++} ++ + } // namespace dataset + } // namespace arrow +diff --git a/cpp/src/arrow/dataset/file_csv.h b/cpp/src/arrow/dataset/file_csv.h +index 42e3fd724..4d2825183 100644 +--- a/cpp/src/arrow/dataset/file_csv.h ++++ b/cpp/src/arrow/dataset/file_csv.h +@@ -85,6 +85,9 @@ class ARROW_DS_EXPORT CsvFileFormat : public FileFormat { + struct ARROW_DS_EXPORT CsvFragmentScanOptions : public FragmentScanOptions { + std::string type_name() const override { return kCsvTypeName; } + ++ static Result> from( ++ const std::unordered_map& configs); ++ + using StreamWrapFunc = std::function>( + std::shared_ptr)>; + +diff --git a/cpp/src/arrow/engine/substrait/expression_internal.cc b/cpp/src/arrow/engine/substrait/expression_internal.cc +index 5d892af9a..0f8b0448b 100644 +--- a/cpp/src/arrow/engine/substrait/expression_internal.cc ++++ b/cpp/src/arrow/engine/substrait/expression_internal.cc +@@ -1337,5 +1337,17 @@ Result> ToProto( + return std::move(out); + } + ++Status FromProto(const substrait::Expression::Literal& literal, ++ std::unordered_map& out) { ++ ARROW_RETURN_IF(!literal.has_map(), Status::Invalid("Literal does not have a map.")); ++ auto literalMap = literal.map(); ++ auto size = literalMap.key_values_size(); ++ for (auto i = 0; i < size; i++) { ++ substrait::Expression_Literal_Map_KeyValue keyValue = literalMap.key_values(i); ++ out.emplace(keyValue.key().string(), keyValue.value().string()); ++ } ++ return Status::OK(); ++} ++ + } // namespace engine + } // namespace arrow +diff --git a/cpp/src/arrow/engine/substrait/expression_internal.h b/cpp/src/arrow/engine/substrait/expression_internal.h +index 2ce2ee76a..9be81b7ab 100644 +--- a/cpp/src/arrow/engine/substrait/expression_internal.h ++++ b/cpp/src/arrow/engine/substrait/expression_internal.h +@@ -61,5 +61,9 @@ ARROW_ENGINE_EXPORT + Result FromProto(const substrait::AggregateFunction&, bool is_hash, + const ExtensionSet&, const ConversionOptions&); + ++ARROW_ENGINE_EXPORT ++Status FromProto(const substrait::Expression::Literal& literal, ++ std::unordered_map& out); ++ + } // namespace engine + } // namespace arrow +diff --git a/cpp/src/arrow/engine/substrait/serde.cc b/cpp/src/arrow/engine/substrait/serde.cc +index 9e670f121..02e5c7171 100644 +--- a/cpp/src/arrow/engine/substrait/serde.cc ++++ b/cpp/src/arrow/engine/substrait/serde.cc +@@ -247,6 +247,16 @@ Result DeserializeExpressions( + return FromProto(extended_expression, ext_set_out, conversion_options, registry); + } + ++Status DeserializeMap(const Buffer& buf, ++ std::unordered_map& out) { ++ // ARROW_ASSIGN_OR_RAISE(auto advanced_extension, ++ // ParseFromBuffer(buf)); ++ // return FromProto(advanced_extension, out); ++ ARROW_ASSIGN_OR_RAISE(auto literal, ++ ParseFromBuffer(buf)); ++ return FromProto(literal, out); ++} ++ + namespace { + + Result> MakeSingleDeclarationPlan( +diff --git a/cpp/src/arrow/engine/substrait/serde.h b/cpp/src/arrow/engine/substrait/serde.h +index ab749f4a6..6312ec239 100644 +--- a/cpp/src/arrow/engine/substrait/serde.h ++++ b/cpp/src/arrow/engine/substrait/serde.h +@@ -23,6 +23,7 @@ + #include + #include + #include ++#include + #include + + #include "arrow/compute/type_fwd.h" +@@ -183,6 +184,9 @@ ARROW_ENGINE_EXPORT Result DeserializeExpressions( + const ConversionOptions& conversion_options = {}, + ExtensionSet* ext_set_out = NULLPTR); + ++ARROW_ENGINE_EXPORT Status ++DeserializeMap(const Buffer& buf, std::unordered_map& out); ++ + /// \brief Deserializes a Substrait Type message to the corresponding Arrow type + /// + /// \param[in] buf a buffer containing the protobuf serialization of a Substrait Type +diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml +index d4d3e2c0f..ce72eaa1f 100644 +--- a/java/dataset/pom.xml ++++ b/java/dataset/pom.xml +@@ -25,9 +25,10 @@ + jar + + ../../../cpp/release-build/ +- 2.5.0 + 1.11.0 + 1.11.3 ++ 0.31.0 ++ 3.25.3 + + + +@@ -47,6 +48,18 @@ + arrow-c-data + compile + ++ ++ io.substrait ++ core ++ ${substrait.version} ++ provided ++ ++ ++ com.google.protobuf ++ protobuf-java ++ ${protobuf.version} ++ provided ++ + + org.apache.arrow + arrow-memory-netty +diff --git a/java/dataset/src/main/cpp/jni_wrapper.cc b/java/dataset/src/main/cpp/jni_wrapper.cc +index 8d7dafd84..89cdc39fe 100644 +--- a/java/dataset/src/main/cpp/jni_wrapper.cc ++++ b/java/dataset/src/main/cpp/jni_wrapper.cc +@@ -25,6 +25,7 @@ + #include "arrow/c/helpers.h" + #include "arrow/dataset/api.h" + #include "arrow/dataset/file_base.h" ++#include "arrow/dataset/file_csv.h" + #include "arrow/filesystem/localfs.h" + #include "arrow/filesystem/path_util.h" + #ifdef ARROW_S3 +@@ -122,6 +123,19 @@ arrow::Result> GetFileFormat( + } + } + ++arrow::Result> ++GetFragmentScanOptions(jint file_format_id, ++ const std::unordered_map& configs) { ++ switch (file_format_id) { ++#ifdef ARROW_CSV ++ case 3: ++ return arrow::dataset::CsvFragmentScanOptions::from(configs); ++#endif ++ default: ++ return arrow::Status::Invalid("Illegal file format id: " ,file_format_id); ++ } ++} ++ + class ReserveFromJava : public arrow::dataset::jni::ReservationListener { + public: + ReserveFromJava(JavaVM* vm, jobject java_reservation_listener) +@@ -460,12 +474,13 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_closeDataset + /* + * Class: org_apache_arrow_dataset_jni_JniWrapper + * Method: createScanner +- * Signature: (J[Ljava/lang/String;Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;JJ)J ++ * Signature: ++ * (J[Ljava/lang/String;Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;JJ;Ljava/nio/ByteBuffer;J)J + */ + JNIEXPORT jlong JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_createScanner( + JNIEnv* env, jobject, jlong dataset_id, jobjectArray columns, +- jobject substrait_projection, jobject substrait_filter, +- jlong batch_size, jlong memory_pool_id) { ++ jobject substrait_projection, jobject substrait_filter, jlong batch_size, ++ jlong file_format_id, jobject options, jlong memory_pool_id) { + JNI_METHOD_START + arrow::MemoryPool* pool = reinterpret_cast(memory_pool_id); + if (pool == nullptr) { +@@ -514,6 +529,14 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_createScann + } + JniAssertOkOrThrow(scanner_builder->Filter(*filter_expr)); + } ++ if (file_format_id != -1 && options != nullptr) { ++ std::unordered_map option_map; ++ std::shared_ptr buffer = LoadArrowBufferFromByteBuffer(env, options); ++ JniAssertOkOrThrow(arrow::engine::DeserializeMap(*buffer, option_map)); ++ std::shared_ptr scan_options = ++ JniGetOrThrow(GetFragmentScanOptions(file_format_id, option_map)); ++ JniAssertOkOrThrow(scanner_builder->FragmentScanOptions(scan_options)); ++ } + JniAssertOkOrThrow(scanner_builder->BatchSize(batch_size)); + + auto scanner = JniGetOrThrow(scanner_builder->Finish()); +@@ -627,14 +650,31 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_ensureS3Fina + /* + * Class: org_apache_arrow_dataset_file_JniWrapper + * Method: makeFileSystemDatasetFactory +- * Signature: (Ljava/lang/String;II)J ++ * Signature: (Ljava/lang/String;IILjava/lang/String;Ljava/nio/ByteBuffer)J + */ + JNIEXPORT jlong JNICALL +-Java_org_apache_arrow_dataset_file_JniWrapper_makeFileSystemDatasetFactory__Ljava_lang_String_2I( +- JNIEnv* env, jobject, jstring uri, jint file_format_id) { ++Java_org_apache_arrow_dataset_file_JniWrapper_makeFileSystemDatasetFactory( ++ JNIEnv* env, jobject, jstring uri, jint file_format_id, jobject options) { + JNI_METHOD_START + std::shared_ptr file_format = + JniGetOrThrow(GetFileFormat(file_format_id)); ++ if (options != nullptr) { ++ std::unordered_map option_map; ++ std::shared_ptr buffer = LoadArrowBufferFromByteBuffer(env, options); ++ JniAssertOkOrThrow(arrow::engine::DeserializeMap(*buffer, option_map)); ++ std::shared_ptr scan_options = ++ JniGetOrThrow(GetFragmentScanOptions(file_format_id, option_map)); ++ file_format->default_fragment_scan_options = scan_options; ++#ifdef ARROW_CSV ++ if (file_format_id == 3) { ++ std::shared_ptr csv_file_format = ++ std::dynamic_pointer_cast(file_format); ++ csv_file_format->parse_options = ++ std::dynamic_pointer_cast(scan_options) ++ ->parse_options; ++ } ++#endif ++ } + arrow::dataset::FileSystemFactoryOptions options; + std::shared_ptr d = + JniGetOrThrow(arrow::dataset::FileSystemDatasetFactory::Make( +@@ -645,16 +685,33 @@ Java_org_apache_arrow_dataset_file_JniWrapper_makeFileSystemDatasetFactory__Ljav + + /* + * Class: org_apache_arrow_dataset_file_JniWrapper +- * Method: makeFileSystemDatasetFactory +- * Signature: ([Ljava/lang/String;II)J ++ * Method: makeFileSystemDatasetFactoryWithFiles ++ * Signature: ([Ljava/lang/String;IIJ;Ljava/nio/ByteBuffer)J + */ + JNIEXPORT jlong JNICALL +-Java_org_apache_arrow_dataset_file_JniWrapper_makeFileSystemDatasetFactory___3Ljava_lang_String_2I( +- JNIEnv* env, jobject, jobjectArray uris, jint file_format_id) { ++Java_org_apache_arrow_dataset_file_JniWrapper_makeFileSystemDatasetFactoryWithFiles( ++ JNIEnv* env, jobject, jobjectArray uris, jint file_format_id, jobject options) { + JNI_METHOD_START + + std::shared_ptr file_format = + JniGetOrThrow(GetFileFormat(file_format_id)); ++ if (options != nullptr) { ++ std::unordered_map option_map; ++ std::shared_ptr buffer = LoadArrowBufferFromByteBuffer(env, options); ++ JniAssertOkOrThrow(arrow::engine::DeserializeMap(*buffer, option_map)); ++ std::shared_ptr scan_options = ++ JniGetOrThrow(GetFragmentScanOptions(file_format_id, option_map)); ++ file_format->default_fragment_scan_options = scan_options; ++#ifdef ARROW_CSV ++ if (file_format_id == 3) { ++ std::shared_ptr csv_file_format = ++ std::dynamic_pointer_cast(file_format); ++ csv_file_format->parse_options = ++ std::dynamic_pointer_cast(scan_options) ++ ->parse_options; ++ } ++#endif ++ } + arrow::dataset::FileSystemFactoryOptions options; + + std::vector uri_vec = ToStringVector(env, uris); +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/file/FileSystemDatasetFactory.java b/java/dataset/src/main/java/org/apache/arrow/dataset/file/FileSystemDatasetFactory.java +index aa3156905..a0b6fb168 100644 +--- a/java/dataset/src/main/java/org/apache/arrow/dataset/file/FileSystemDatasetFactory.java ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/file/FileSystemDatasetFactory.java +@@ -17,8 +17,11 @@ + + package org.apache.arrow.dataset.file; + ++import java.util.Optional; ++ + import org.apache.arrow.dataset.jni.NativeDatasetFactory; + import org.apache.arrow.dataset.jni.NativeMemoryPool; ++import org.apache.arrow.dataset.scanner.FragmentScanOptions; + import org.apache.arrow.memory.BufferAllocator; + + /** +@@ -27,21 +30,34 @@ import org.apache.arrow.memory.BufferAllocator; + public class FileSystemDatasetFactory extends NativeDatasetFactory { + + public FileSystemDatasetFactory(BufferAllocator allocator, NativeMemoryPool memoryPool, FileFormat format, +- String uri) { +- super(allocator, memoryPool, createNative(format, uri)); ++ String uri, Optional fragmentScanOptions) { ++ super(allocator, memoryPool, createNative(format, uri, fragmentScanOptions)); ++ } ++ ++ public FileSystemDatasetFactory(BufferAllocator allocator, NativeMemoryPool memoryPool, FileFormat format, ++ String uri) { ++ super(allocator, memoryPool, createNative(format, uri, Optional.empty())); ++ } ++ ++ public FileSystemDatasetFactory(BufferAllocator allocator, NativeMemoryPool memoryPool, FileFormat format, ++ String[] uris, Optional fragmentScanOptions) { ++ super(allocator, memoryPool, createNative(format, uris, fragmentScanOptions)); + } + + public FileSystemDatasetFactory(BufferAllocator allocator, NativeMemoryPool memoryPool, FileFormat format, + String[] uris) { +- super(allocator, memoryPool, createNative(format, uris)); ++ super(allocator, memoryPool, createNative(format, uris, Optional.empty())); + } + +- private static long createNative(FileFormat format, String uri) { +- return JniWrapper.get().makeFileSystemDatasetFactory(uri, format.id()); ++ private static long createNative(FileFormat format, String uri, Optional fragmentScanOptions) { ++ return JniWrapper.get().makeFileSystemDatasetFactory(uri, format.id(), ++ fragmentScanOptions.map(FragmentScanOptions::serialize).orElse(null)); + } + +- private static long createNative(FileFormat format, String[] uris) { +- return JniWrapper.get().makeFileSystemDatasetFactory(uris, format.id()); ++ private static long createNative(FileFormat format, String[] uris, ++ Optional fragmentScanOptions) { ++ return JniWrapper.get().makeFileSystemDatasetFactoryWithFiles(uris, format.id(), ++ fragmentScanOptions.map(FragmentScanOptions::serialize).orElse(null)); + } + + } +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/file/JniWrapper.java b/java/dataset/src/main/java/org/apache/arrow/dataset/file/JniWrapper.java +index c3a1a4e58..c3f8e12b3 100644 +--- a/java/dataset/src/main/java/org/apache/arrow/dataset/file/JniWrapper.java ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/file/JniWrapper.java +@@ -17,6 +17,8 @@ + + package org.apache.arrow.dataset.file; + ++import java.nio.ByteBuffer; ++ + import org.apache.arrow.dataset.jni.JniLoader; + + /** +@@ -43,7 +45,8 @@ public class JniWrapper { + * @return the native pointer of the arrow::dataset::FileSystemDatasetFactory instance. + * @see FileFormat + */ +- public native long makeFileSystemDatasetFactory(String uri, int fileFormat); ++ public native long makeFileSystemDatasetFactory(String uri, int fileFormat, ++ ByteBuffer serializedFragmentScanOptions); + + /** + * Create FileSystemDatasetFactory and return its native pointer. The pointer is pointing to a +@@ -54,7 +57,8 @@ public class JniWrapper { + * @return the native pointer of the arrow::dataset::FileSystemDatasetFactory instance. + * @see FileFormat + */ +- public native long makeFileSystemDatasetFactory(String[] uris, int fileFormat); ++ public native long makeFileSystemDatasetFactoryWithFiles(String[] uris, int fileFormat, ++ ByteBuffer serializedFragmentScanOptions); + + /** + * Write the content in a {@link org.apache.arrow.c.ArrowArrayStream} into files. This internally +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java +index 637a3e8f2..6d6309140 100644 +--- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniWrapper.java +@@ -80,7 +80,8 @@ public class JniWrapper { + * @return the native pointer of the arrow::dataset::Scanner instance. + */ + public native long createScanner(long datasetId, String[] columns, ByteBuffer substraitProjection, +- ByteBuffer substraitFilter, long batchSize, long memoryPool); ++ ByteBuffer substraitFilter, long batchSize, long fileFormat, ++ ByteBuffer serializedFragmentScanOptions, long memoryPool); + + /** + * Get a serialized schema from native instance of a Scanner. +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeDataset.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeDataset.java +index d9abad997..3a96fe768 100644 +--- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeDataset.java ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeDataset.java +@@ -17,6 +17,9 @@ + + package org.apache.arrow.dataset.jni; + ++import java.nio.ByteBuffer; ++ ++import org.apache.arrow.dataset.scanner.FragmentScanOptions; + import org.apache.arrow.dataset.scanner.ScanOptions; + import org.apache.arrow.dataset.source.Dataset; + +@@ -40,11 +43,18 @@ public class NativeDataset implements Dataset { + if (closed) { + throw new NativeInstanceReleasedException(); + } +- ++ int fileFormat = -1; ++ ByteBuffer serialized = null; ++ if (options.getFragmentScanOptions().isPresent()) { ++ FragmentScanOptions fragmentScanOptions = options.getFragmentScanOptions().get(); ++ fileFormat = fragmentScanOptions.fileFormatId(); ++ serialized = fragmentScanOptions.serialize(); ++ } + long scannerId = JniWrapper.get().createScanner(datasetId, options.getColumns().orElse(null), + options.getSubstraitProjection().orElse(null), + options.getSubstraitFilter().orElse(null), +- options.getBatchSize(), context.getMemoryPool().getNativeInstanceId()); ++ options.getBatchSize(), fileFormat, serialized, ++ context.getMemoryPool().getNativeInstanceId()); + + return new NativeScanner(context, scannerId); + } +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/FragmentScanOptions.java b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/FragmentScanOptions.java +new file mode 100644 +index 000000000..8acb2b2d4 +--- /dev/null ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/FragmentScanOptions.java +@@ -0,0 +1,50 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.arrow.dataset.scanner; ++ ++import java.nio.ByteBuffer; ++import java.util.Map; ++ ++import org.apache.arrow.dataset.substrait.util.ConvertUtil; ++ ++import io.substrait.proto.Expression; ++ ++public interface FragmentScanOptions { ++ String typeName(); ++ ++ int fileFormatId(); ++ ++ ByteBuffer serialize(); ++ ++ /** ++ * serialize the map. ++ * ++ * @param config config map ++ * @return bufer to jni call argument, should be DirectByteBuffer ++ */ ++ default ByteBuffer serializeMap(Map config) { ++ if (config.isEmpty()) { ++ return null; ++ } ++ ++ Expression.Literal literal = ConvertUtil.mapToExpressionLiteral(config); ++ ByteBuffer buf = ByteBuffer.allocateDirect(literal.getSerializedSize()); ++ buf.put(literal.toByteArray()); ++ return buf; ++ } ++} +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/ScanOptions.java b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/ScanOptions.java +index 995d05ac3..aad71930c 100644 +--- a/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/ScanOptions.java ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/ScanOptions.java +@@ -31,6 +31,8 @@ public class ScanOptions { + private final Optional substraitProjection; + private final Optional substraitFilter; + ++ private final Optional fragmentScanOptions; ++ + /** + * Constructor. + * @param columns Projected columns. Empty for scanning all columns. +@@ -61,6 +63,7 @@ public class ScanOptions { + this.columns = columns; + this.substraitProjection = Optional.empty(); + this.substraitFilter = Optional.empty(); ++ this.fragmentScanOptions = Optional.empty(); + } + + public ScanOptions(long batchSize) { +@@ -83,6 +86,10 @@ public class ScanOptions { + return substraitFilter; + } + ++ public Optional getFragmentScanOptions() { ++ return fragmentScanOptions; ++ } ++ + /** + * Builder for Options used during scanning. + */ +@@ -91,6 +98,7 @@ public class ScanOptions { + private Optional columns; + private ByteBuffer substraitProjection; + private ByteBuffer substraitFilter; ++ private FragmentScanOptions fragmentScanOptions; + + /** + * Constructor. +@@ -136,6 +144,18 @@ public class ScanOptions { + return this; + } + ++ /** ++ * Set the FragmentScanOptions. ++ * ++ * @param fragmentScanOptions scan options ++ * @return the ScanOptions configured. ++ */ ++ public Builder fragmentScanOptions(FragmentScanOptions fragmentScanOptions) { ++ Preconditions.checkNotNull(fragmentScanOptions); ++ this.fragmentScanOptions = fragmentScanOptions; ++ return this; ++ } ++ + public ScanOptions build() { + return new ScanOptions(this); + } +@@ -146,5 +166,6 @@ public class ScanOptions { + columns = builder.columns; + substraitProjection = Optional.ofNullable(builder.substraitProjection); + substraitFilter = Optional.ofNullable(builder.substraitFilter); ++ fragmentScanOptions = Optional.ofNullable(builder.fragmentScanOptions); + } + } +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/csv/CsvConvertOptions.java b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/csv/CsvConvertOptions.java +new file mode 100644 +index 000000000..08e35ede2 +--- /dev/null ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/csv/CsvConvertOptions.java +@@ -0,0 +1,51 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.arrow.dataset.scanner.csv; ++ ++import java.util.Map; ++import java.util.Optional; ++ ++import org.apache.arrow.c.ArrowSchema; ++ ++public class CsvConvertOptions { ++ ++ private final Map configs; ++ ++ private Optional cSchema = Optional.empty(); ++ ++ public CsvConvertOptions(Map configs) { ++ this.configs = configs; ++ } ++ ++ public Optional getArrowSchema() { ++ return cSchema; ++ } ++ ++ public Map getConfigs() { ++ return configs; ++ } ++ ++ public void set(String key, String value) { ++ configs.put(key, value); ++ } ++ ++ public void setArrowSchema(ArrowSchema cSchema) { ++ this.cSchema = Optional.of(cSchema); ++ } ++ ++} +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/csv/CsvFragmentScanOptions.java b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/csv/CsvFragmentScanOptions.java +new file mode 100644 +index 000000000..88973f0a0 +--- /dev/null ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/scanner/csv/CsvFragmentScanOptions.java +@@ -0,0 +1,97 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.arrow.dataset.scanner.csv; ++ ++import java.io.Serializable; ++import java.nio.ByteBuffer; ++import java.util.Locale; ++import java.util.Map; ++import java.util.stream.Collectors; ++import java.util.stream.Stream; ++ ++import org.apache.arrow.dataset.file.FileFormat; ++import org.apache.arrow.dataset.scanner.FragmentScanOptions; ++ ++public class CsvFragmentScanOptions implements Serializable, FragmentScanOptions { ++ private final CsvConvertOptions convertOptions; ++ private final Map readOptions; ++ private final Map parseOptions; ++ ++ ++ /** ++ * csv scan options, map to CPP struct CsvFragmentScanOptions. ++ * ++ * @param convertOptions same struct in CPP ++ * @param readOptions same struct in CPP ++ * @param parseOptions same struct in CPP ++ */ ++ public CsvFragmentScanOptions(CsvConvertOptions convertOptions, ++ Map readOptions, ++ Map parseOptions) { ++ this.convertOptions = convertOptions; ++ this.readOptions = readOptions; ++ this.parseOptions = parseOptions; ++ } ++ ++ public String typeName() { ++ return FileFormat.CSV.name().toLowerCase(Locale.ROOT); ++ } ++ ++ /** ++ * File format id. ++ * ++ * @return id ++ */ ++ public int fileFormatId() { ++ return FileFormat.CSV.id(); ++ } ++ ++ /** ++ * Serialize this class to ByteBuffer and then called by jni call. ++ * ++ * @return DirectByteBuffer ++ */ ++ public ByteBuffer serialize() { ++ Map options = Stream.concat(Stream.concat(readOptions.entrySet().stream(), ++ parseOptions.entrySet().stream()), ++ convertOptions.getConfigs().entrySet().stream()).collect( ++ Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); ++ ++ if (convertOptions.getArrowSchema().isPresent()) { ++ options.put("column_types", Long.toString(convertOptions.getArrowSchema().get().memoryAddress())); ++ } ++ return serializeMap(options); ++ } ++ ++ public static CsvFragmentScanOptions deserialize(String serialized) { ++ throw new UnsupportedOperationException("Not implemented now"); ++ } ++ ++ public CsvConvertOptions getConvertOptions() { ++ return convertOptions; ++ } ++ ++ public Map getReadOptions() { ++ return readOptions; ++ } ++ ++ public Map getParseOptions() { ++ return parseOptions; ++ } ++ ++} +diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/substrait/util/ConvertUtil.java b/java/dataset/src/main/java/org/apache/arrow/dataset/substrait/util/ConvertUtil.java +new file mode 100644 +index 000000000..31a4023af +--- /dev/null ++++ b/java/dataset/src/main/java/org/apache/arrow/dataset/substrait/util/ConvertUtil.java +@@ -0,0 +1,46 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.arrow.dataset.substrait.util; ++ ++import java.util.Map; ++ ++import io.substrait.proto.Expression; ++ ++public class ConvertUtil { ++ ++ /** ++ * Convert map to substrait Expression. ++ * ++ * @return Substrait Expression ++ */ ++ public static Expression.Literal mapToExpressionLiteral(Map values) { ++ Expression.Literal.Builder literalBuilder = Expression.Literal.newBuilder(); ++ Expression.Literal.Map.KeyValue.Builder keyValueBuilder = ++ Expression.Literal.Map.KeyValue.newBuilder(); ++ Expression.Literal.Map.Builder mapBuilder = Expression.Literal.Map.newBuilder(); ++ for (Map.Entry entry : values.entrySet()) { ++ literalBuilder.setString(entry.getKey()); ++ keyValueBuilder.setKey(literalBuilder.build()); ++ literalBuilder.setString(entry.getValue()); ++ keyValueBuilder.setValue(literalBuilder.build()); ++ mapBuilder.addKeyValues(keyValueBuilder.build()); ++ } ++ literalBuilder.setMap(mapBuilder.build()); ++ return literalBuilder.build(); ++ } ++} +diff --git a/java/dataset/src/test/java/org/apache/arrow/dataset/substrait/TestAceroSubstraitConsumer.java b/java/dataset/src/test/java/org/apache/arrow/dataset/substrait/TestAceroSubstraitConsumer.java +index 0fba72892..e7903b7a4 100644 +--- a/java/dataset/src/test/java/org/apache/arrow/dataset/substrait/TestAceroSubstraitConsumer.java ++++ b/java/dataset/src/test/java/org/apache/arrow/dataset/substrait/TestAceroSubstraitConsumer.java +@@ -31,6 +31,9 @@ import java.util.HashMap; + import java.util.Map; + import java.util.Optional; + ++import org.apache.arrow.c.ArrowSchema; ++import org.apache.arrow.c.CDataDictionaryProvider; ++import org.apache.arrow.c.Data; + import org.apache.arrow.dataset.ParquetWriteSupport; + import org.apache.arrow.dataset.TestDataset; + import org.apache.arrow.dataset.file.FileFormat; +@@ -38,8 +41,11 @@ import org.apache.arrow.dataset.file.FileSystemDatasetFactory; + import org.apache.arrow.dataset.jni.NativeMemoryPool; + import org.apache.arrow.dataset.scanner.ScanOptions; + import org.apache.arrow.dataset.scanner.Scanner; ++import org.apache.arrow.dataset.scanner.csv.CsvConvertOptions; ++import org.apache.arrow.dataset.scanner.csv.CsvFragmentScanOptions; + import org.apache.arrow.dataset.source.Dataset; + import org.apache.arrow.dataset.source.DatasetFactory; ++import org.apache.arrow.memory.BufferAllocator; + import org.apache.arrow.vector.ipc.ArrowReader; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; +@@ -49,6 +55,8 @@ import org.junit.ClassRule; + import org.junit.Test; + import org.junit.rules.TemporaryFolder; + ++import com.google.common.collect.ImmutableMap; ++ + public class TestAceroSubstraitConsumer extends TestDataset { + + @ClassRule +@@ -457,4 +465,42 @@ public class TestAceroSubstraitConsumer extends TestDataset { + substraitExpression.put(decodedSubstrait); + return substraitExpression; + } ++ ++ @Test ++ public void testCsvConvertOptions() throws Exception { ++ final Schema schema = new Schema(Arrays.asList( ++ Field.nullable("Id", new ArrowType.Int(32, true)), ++ Field.nullable("Name", new ArrowType.Utf8()), ++ Field.nullable("Language", new ArrowType.Utf8()) ++ ), null); ++ String path = "file://" + getClass().getResource("/").getPath() + "/data/student.csv"; ++ BufferAllocator allocator = rootAllocator(); ++ try (ArrowSchema cSchema = ArrowSchema.allocateNew(allocator); ++ CDataDictionaryProvider provider = new CDataDictionaryProvider()) { ++ Data.exportSchema(allocator, schema, provider, cSchema); ++ CsvConvertOptions convertOptions = new CsvConvertOptions(ImmutableMap.of("delimiter", ";")); ++ convertOptions.setArrowSchema(cSchema); ++ CsvFragmentScanOptions fragmentScanOptions = new CsvFragmentScanOptions( ++ convertOptions, ImmutableMap.of(), ImmutableMap.of()); ++ ScanOptions options = new ScanOptions.Builder(/*batchSize*/ 32768) ++ .columns(Optional.empty()) ++ .fragmentScanOptions(fragmentScanOptions) ++ .build(); ++ try ( ++ DatasetFactory datasetFactory = new FileSystemDatasetFactory(allocator, NativeMemoryPool.getDefault(), ++ FileFormat.CSV, path); ++ Dataset dataset = datasetFactory.finish(); ++ Scanner scanner = dataset.newScan(options); ++ ArrowReader reader = scanner.scanBatches() ++ ) { ++ assertEquals(schema.getFields(), reader.getVectorSchemaRoot().getSchema().getFields()); ++ int rowCount = 0; ++ while (reader.loadNextBatch()) { ++ assertEquals("[1, 2, 3]", reader.getVectorSchemaRoot().getVector("Id").toString()); ++ rowCount += reader.getVectorSchemaRoot().getRowCount(); ++ } ++ assertEquals(3, rowCount); ++ } ++ } ++ } + } +diff --git a/java/dataset/src/test/resources/data/student.csv b/java/dataset/src/test/resources/data/student.csv +new file mode 100644 +index 000000000..329194609 +--- /dev/null ++++ b/java/dataset/src/test/resources/data/student.csv +@@ -0,0 +1,4 @@ ++Id;Name;Language ++1;Juno;Java ++2;Peter;Python ++3;Celin;C++ diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch index 4bcb228455b1..09af35020842 100644 --- a/ep/build-velox/src/modify_velox.patch +++ b/ep/build-velox/src/modify_velox.patch @@ -83,11 +83,12 @@ index ce4c24dbe..785a2acc6 100644 -DARROW_WITH_THRIFT=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -@@ -69,6 +71,7 @@ if(VELOX_ENABLE_ARROW) +@@ -69,6 +71,8 @@ if(VELOX_ENABLE_ARROW) arrow_ep PREFIX ${ARROW_PREFIX} URL ${VELOX_ARROW_SOURCE_URL} + PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow.patch ++ COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM} SOURCE_SUBDIR cpp CMAKE_ARGS ${ARROW_CMAKE_ARGS} diff --git a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index 7d2d48828fb3..95391a2c42f5 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -319,6 +319,26 @@ abstract class WholeStageTransformerSuite df } + /** + * Some rule on LogicalPlan will not only apply in select query, the total df.load() should in + * spark environment with gluten disabled config. + * + * @param sql + * @param f + * @return + */ + protected def runAndCompare(sql: String)(f: => Unit): DataFrame = { + var expected: Seq[Row] = null + withSQLConf(vanillaSparkConfs(): _*) { + f + expected = spark.sql(sql).collect() + } + f + val df = spark.sql(sql) + checkAnswer(df, expected) + df + } + protected def runQueryAndCompare( sqlStr: String, compareResult: Boolean = true, diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml index bb84a06b4125..c28490d77faa 100644 --- a/gluten-data/pom.xml +++ b/gluten-data/pom.xml @@ -124,7 +124,7 @@ org.apache.arrow arrow-c-data - ${arrow.version} + ${arrow-gluten.version} org.apache.arrow @@ -140,7 +140,7 @@ org.apache.arrow arrow-dataset - ${arrow.version} + ${arrow-gluten.version} io.netty diff --git a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala index 99eb72c70ea3..a94f8f2e3d49 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/utils/ArrowUtil.scala @@ -16,16 +16,12 @@ */ package org.apache.gluten.utils -import org.apache.gluten.exception.SchemaMismatchException import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat -import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.vectorized.ArrowColumnVectorUtils -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.sql.utils.{SparkArrowUtil, SparkSchemaUtil} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} @@ -33,13 +29,15 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.arrow.c.{ArrowSchema, CDataDictionaryProvider, Data} import org.apache.arrow.dataset.file.{FileFormat, FileSystemDatasetFactory} import org.apache.arrow.dataset.jni.NativeMemoryPool +import org.apache.arrow.dataset.scanner.FragmentScanOptions import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.ipc.message.ArrowRecordBatch import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} import org.apache.hadoop.fs.FileStatus -import java.net.URI +import java.net.{URI, URLDecoder} import java.util +import java.util.Optional import scala.collection.JavaConverters._ import scala.collection.mutable @@ -99,26 +97,6 @@ object ArrowUtil extends Logging { new Schema(fields) } - def getFormat(format: String): FileFormat = { - format match { - case "parquet" => FileFormat.PARQUET - case "orc" => FileFormat.ORC - case "csv" => FileFormat.CSV - case _ => throw new IllegalArgumentException("Unrecognizable format") - } - } - - def getFormat(format: org.apache.spark.sql.execution.datasources.FileFormat): FileFormat = { - format match { - case _: ParquetFileFormat => - FileFormat.PARQUET - case _: CSVFileFormat => - FileFormat.CSV - case _ => - throw new IllegalArgumentException("Unrecognizable format") - } - } - private def rewriteUri(encodeUri: String): String = { val decodedUri = encodeUri val uri = URI.create(decodedUri) @@ -142,19 +120,49 @@ object ArrowUtil extends Logging { def makeArrowDiscovery( encodedUri: String, format: FileFormat, + option: Optional[FragmentScanOptions], allocator: BufferAllocator, - pool: NativeMemoryPool): FileSystemDatasetFactory = { - val factory = new FileSystemDatasetFactory(allocator, pool, format, rewriteUri(encodedUri)) + pool: NativeMemoryPool + ): FileSystemDatasetFactory = { + val factory = + new FileSystemDatasetFactory(allocator, pool, format, rewriteUri(encodedUri), option) factory } + def readArrowSchema( + file: String, + format: FileFormat, + option: FragmentScanOptions, + allocator: BufferAllocator, + pool: NativeMemoryPool): Schema = { + val factory: FileSystemDatasetFactory = + makeArrowDiscovery(file, format, Optional.of(option), allocator, pool) + val schema = factory.inspect() + factory.close() + schema + } + + def readArrowFileColumnNames( + file: String, + format: FileFormat, + option: FragmentScanOptions, + allocator: BufferAllocator, + pool: NativeMemoryPool): Array[String] = { + val fileFields = ArrowUtil + .readArrowSchema(URLDecoder.decode(file, "UTF-8"), format, option, allocator, pool) + .getFields + .asScala + fileFields.map(_.getName).toArray + } + def readSchema( file: FileStatus, format: FileFormat, + option: FragmentScanOptions, allocator: BufferAllocator, pool: NativeMemoryPool): Option[StructType] = { val factory: FileSystemDatasetFactory = - makeArrowDiscovery(file.getPath.toString, format, allocator, pool) + makeArrowDiscovery(file.getPath.toString, format, Optional.of(option), allocator, pool) val schema = factory.inspect() try { Option(SparkSchemaUtil.fromArrowSchema(schema)) @@ -166,67 +174,14 @@ object ArrowUtil extends Logging { def readSchema( files: Seq[FileStatus], format: FileFormat, + option: FragmentScanOptions, allocator: BufferAllocator, pool: NativeMemoryPool): Option[StructType] = { if (files.isEmpty) { throw new IllegalArgumentException("No input file specified") } - readSchema(files.head, format, allocator, pool) - } - - def compareStringFunc(caseSensitive: Boolean): (String, String) => Boolean = { - if (caseSensitive) { (str1: String, str2: String) => str1.equals(str2) } - else { (str1: String, str2: String) => str1.equalsIgnoreCase(str2) } - } - - // If user specify schema by .schema(newSchemaDifferentWithFile) - def checkSchema( - requiredField: DataType, - parquetFileFieldType: ArrowType, - parquetFileFields: mutable.Buffer[Field]): Unit = { - val requiredFieldType = - SparkArrowUtil.toArrowType(requiredField, SparkSchemaUtil.getLocalTimezoneID) - if (!requiredFieldType.equals(parquetFileFieldType)) { - val arrowFileSchema = parquetFileFields - .map(f => f.toString) - .reduceLeft((f1, f2) => f1 + "\n" + f2) - throw new SchemaMismatchException( - s"Not support specified schema is different with file schema\n$arrowFileSchema") - } - } - - def getRequestedField( - requiredSchema: StructType, - parquetFileFields: mutable.Buffer[Field], - caseSensitive: Boolean): Schema = { - val compareFunc = compareStringFunc(caseSensitive) - requiredSchema.foreach { - readField => - // TODO: check schema inside of complex type - val matchedFields = - parquetFileFields.filter(field => compareFunc(field.getName, readField.name)) - if (!caseSensitive && matchedFields.size > 1) { - // Need to fail if there is ambiguity, i.e. more than one field is matched - val fieldsString = matchedFields.map(_.getName).mkString("[", ", ", "]") - throw new RuntimeException( - s""" - |Found duplicate field(s) "${readField.name}": $fieldsString - - |in case-insensitive mode""".stripMargin.replaceAll("\n", " ")) - } - if (matchedFields.nonEmpty) { - checkSchema( - readField.dataType, - matchedFields.head.getFieldType.getType, - parquetFileFields) - } - } - - val requestColNames = requiredSchema.map(_.name) - new Schema(parquetFileFields.filter { - field => requestColNames.exists(col => compareFunc(col, field.getName)) - }.asJava) + readSchema(files.head, format, option, allocator, pool) } def loadMissingColumns( @@ -262,19 +217,14 @@ object ArrowUtil extends Logging { def loadBatch( allocator: BufferAllocator, input: ArrowRecordBatch, - dataSchema: StructType, - requiredSchema: StructType, + dataSchema: Schema, partitionVectors: Array[ArrowWritableColumnVector] = Array.empty, nullVectors: Array[ArrowWritableColumnVector] = Array.empty): ColumnarBatch = { val rowCount: Int = input.getLength val vectors = try { - ArrowWritableColumnVector.loadColumns( - rowCount, - SparkSchemaUtil.toArrowSchema(dataSchema), - input, - allocator) + ArrowWritableColumnVector.loadColumns(rowCount, dataSchema, input, allocator) } finally { input.close() } @@ -282,21 +232,8 @@ object ArrowUtil extends Logging { val totalVectors = if (nullVectors.nonEmpty) { val finalVectors = mutable.ArrayBuffer[ArrowWritableColumnVector]() - val requiredIterator = requiredSchema.iterator - val compareFunc = compareStringFunc(SQLConf.get.caseSensitiveAnalysis) - while (requiredIterator.hasNext) { - val field = requiredIterator.next() - finalVectors.append(vectors - .find(vector => compareFunc(vector.getValueVector.getName, field.name)) - .getOrElse { - // The missing column need to be find in nullVectors - val nullVector = - nullVectors.find(vector => compareFunc(vector.getValueVector.getName, field.name)).get - nullVector.setValueCount(rowCount) - nullVector.retain() - nullVector - }) - } + finalVectors.appendAll(vectors) + finalVectors.appendAll(nullVectors) finalVectors.toArray } else { vectors diff --git a/gluten-ut/spark32/pom.xml b/gluten-ut/spark32/pom.xml index e026bb424129..b0744589d161 100644 --- a/gluten-ut/spark32/pom.xml +++ b/gluten-ut/spark32/pom.xml @@ -76,7 +76,7 @@ org.apache.arrow arrow-c-data - ${arrow.version} + ${arrow-gluten.version} test diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 664cd37d1f7e..5762855c2cde 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -419,6 +419,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] @@ -431,6 +434,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch // Early Filter and Projection Push-Down generated an invalid plan .exclude("SPARK-26208: write and read empty data to csv file with headers") @@ -443,6 +449,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") enableSuite[GlutenJsonV1Suite] // FIXME: Array direct selection fails .exclude("Complex field and type inferring") diff --git a/gluten-ut/spark33/pom.xml b/gluten-ut/spark33/pom.xml index 0f0a0703c3c5..5f9a28e2459d 100644 --- a/gluten-ut/spark33/pom.xml +++ b/gluten-ut/spark33/pom.xml @@ -83,7 +83,7 @@ org.apache.arrow arrow-c-data - ${arrow.version} + ${arrow-gluten.version} test diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 57c1976221df..898fc2b39583 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -206,6 +206,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] @@ -218,6 +221,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch // Early Filter and Projection Push-Down generated an invalid plan .exclude("SPARK-26208: write and read empty data to csv file with headers") @@ -229,6 +235,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenJsonV1Suite] diff --git a/gluten-ut/spark34/pom.xml b/gluten-ut/spark34/pom.xml index d30f9644dbb4..a8d24d5fd219 100644 --- a/gluten-ut/spark34/pom.xml +++ b/gluten-ut/spark34/pom.xml @@ -83,7 +83,7 @@ org.apache.arrow arrow-c-data - ${arrow.version} + ${arrow-gluten.version} test diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 6126844ade08..d8e3a5ecc051 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -184,6 +184,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] @@ -196,6 +199,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Rule org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown in batch // Early Filter and Projection Push-Down generated an invalid plan .exclude("SPARK-26208: write and read empty data to csv file with headers") @@ -207,6 +213,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenJsonV1Suite] diff --git a/gluten-ut/spark35/pom.xml b/gluten-ut/spark35/pom.xml index 8f0a5605bdff..cf2129389a6e 100644 --- a/gluten-ut/spark35/pom.xml +++ b/gluten-ut/spark35/pom.xml @@ -95,7 +95,7 @@ org.apache.arrow arrow-c-data - ${arrow.version} + ${arrow-gluten.version} test diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 28f334878689..10f7be4feaeb 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -187,6 +187,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVv2Suite] @@ -202,6 +205,11 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("save csv with empty fields with user defined empty values") .exclude("save csv with quote") .exclude("SPARK-13543 Write the output as uncompressed via option()") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") + // Arrow not support corrupt record + .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") enableSuite[GlutenCSVLegacyTimeParserSuite] // file cars.csv include null string, Arrow not support to read .exclude("DDL test with schema") @@ -212,6 +220,9 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-13543 Write the output as uncompressed via option()") // Arrow not support corrupt record .exclude("SPARK-27873: disabling enforceSchema should not fail columnNameOfCorruptRecord") + .exclude("DDL test with tab separated file") + .exclude("DDL test parsing decimal type") + .exclude("test with tab delimiter and double quote") enableSuite[GlutenJsonV1Suite] // FIXME: Array direct selection fails .exclude("Complex field and type inferring") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala index cb7ce87f97da..8b75dad33c38 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala @@ -20,16 +20,14 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.spark.{SparkConf, SparkException} -import org.apache.spark.sql.{AnalysisException, GlutenSQLTestsBaseTrait, Row} +import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, Row} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DateType, IntegerType, StringType, StructType, TimestampType} +import org.apache.spark.sql.types.{DateType, IntegerType, StructType, TimestampType} import org.scalatest.exceptions.TestFailedException import java.sql.{Date, Timestamp} -import scala.collection.JavaConverters.seqAsJavaListConverter - class GlutenCSVSuite extends CSVSuite with GlutenSQLTestsBaseTrait { override def sparkConf: SparkConf = @@ -43,68 +41,9 @@ class GlutenCSVSuite extends CSVSuite with GlutenSQLTestsBaseTrait { } class GlutenCSVv1Suite extends GlutenCSVSuite { - import testImplicits._ override def sparkConf: SparkConf = super.sparkConf .set(SQLConf.USE_V1_SOURCE_LIST, "csv") - - testGluten("SPARK-23786: Ignore column name case if spark.sql.caseSensitive is false") { - withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { - withTempPath { - path => - val oschema = new StructType().add("A", StringType) - // change the row content 0 to string bbb in Gluten for test - val odf = spark.createDataFrame(List(Row("bbb")).asJava, oschema) - odf.write.option("header", true).csv(path.getCanonicalPath) - val ischema = new StructType().add("a", StringType) - val idf = spark.read - .schema(ischema) - .option("header", true) - .option("enforceSchema", false) - .csv(path.getCanonicalPath) - checkAnswer(idf, odf) - } - } - } - - testGluten("case sensitivity of filters references") { - Seq(true, false).foreach { - filterPushdown => - withSQLConf(SQLConf.CSV_FILTER_PUSHDOWN_ENABLED.key -> filterPushdown.toString) { - withTempPath { - path => - Seq("""aaa,BBB""", """0,1""", """2,3""") - .toDF() - .repartition(1) - .write - .text(path.getCanonicalPath) - withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { - // change the schema to Arrow schema to support read in Gluten - val readback = spark.read - .schema("aaa long, BBB long") - .option("header", true) - .csv(path.getCanonicalPath) - checkAnswer(readback, Seq(Row(2, 3), Row(0, 1))) - checkAnswer(readback.filter($"AAA" === 2 && $"bbb" === 3), Seq(Row(2, 3))) - } - withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") { - val readback = spark.read - .schema("aaa long, BBB long") - .option("header", true) - .csv(path.getCanonicalPath) - checkAnswer(readback, Seq(Row(2, 3), Row(0, 1))) - checkError( - exception = intercept[AnalysisException] { - readback.filter($"AAA" === 2 && $"bbb" === 3).collect() - }, - errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION", - parameters = Map("objectName" -> "`AAA`", "proposal" -> "`BBB`, `aaa`") - ) - } - } - } - } - } } class GlutenCSVv2Suite extends GlutenCSVSuite { diff --git a/gluten-ut/test/pom.xml b/gluten-ut/test/pom.xml index d55e6ca917e7..25ec542deab2 100644 --- a/gluten-ut/test/pom.xml +++ b/gluten-ut/test/pom.xml @@ -90,7 +90,7 @@ org.apache.arrow arrow-c-data - ${arrow.version} + ${arrow-gluten.version} test diff --git a/pom.xml b/pom.xml index 63c53f109302..88cbb724e053 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,7 @@ 0.3.2-incubating 0.8.0 15.0.0 + 15.0.0-gluten arrow-memory-unsafe 2.7.4 UTF-8 From 4dcda6a2510ccfa468748fe1e1181a9e2ad3500e Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Mon, 3 Jun 2024 11:33:06 +0800 Subject: [PATCH 195/402] [VL] Upgrade simdjson to 3.9.3 in vcpkg build (#5938) --- dev/ci-velox-buildstatic.sh | 2 +- dev/vcpkg/ports/simdjson/portfile.cmake | 44 +++++++++++++++++++++++++ dev/vcpkg/ports/simdjson/vcpkg.json | 37 +++++++++++++++++++++ dev/vcpkg/vcpkg.json | 3 +- ep/build-velox/src/build_velox.sh | 2 +- 5 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 dev/vcpkg/ports/simdjson/portfile.cmake create mode 100644 dev/vcpkg/ports/simdjson/vcpkg.json diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic.sh index 208490d1c2eb..227bad36053b 100755 --- a/dev/ci-velox-buildstatic.sh +++ b/dev/ci-velox-buildstatic.sh @@ -2,8 +2,8 @@ yum install sudo patch java-1.8.0-openjdk-devel -y cd $GITHUB_WORKSPACE/ep/build-velox/src ./get_velox.sh source /opt/rh/devtoolset-9/enable -source /opt/gluten/dev/vcpkg/env.sh cd $GITHUB_WORKSPACE/ +source ./dev/vcpkg/env.sh sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt export NUM_THREADS=4 ./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON diff --git a/dev/vcpkg/ports/simdjson/portfile.cmake b/dev/vcpkg/ports/simdjson/portfile.cmake new file mode 100644 index 000000000000..ecc2321c13e1 --- /dev/null +++ b/dev/vcpkg/ports/simdjson/portfile.cmake @@ -0,0 +1,44 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO simdjson/simdjson + REF "v${VERSION}" + HEAD_REF master + SHA512 893ce0cb380b1418438f5910262325031f8071c4577589a491713f91c980964b4105c1e1aa7f2b9373deed40ecf6b48fe48a1aa243151e424f138f5418e4821c +) + +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + exceptions SIMDJSON_EXCEPTIONS + threads SIMDJSON_ENABLE_THREADS + INVERTED_FEATURES + deprecated SIMDJSON_DISABLE_DEPRECATED_API + utf8-validation SIMDJSON_SKIPUTF8VALIDATION +) + +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" SIMDJSON_BUILD_STATIC) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + -DSIMDJSON_JUST_LIBRARY=ON + -DSIMDJSON_SANITIZE_UNDEFINED=OFF + -DSIMDJSON_SANITIZE=OFF + -DSIMDJSON_SANITIZE_THREADS=OFF + -DSIMDJSON_BUILD_STATIC=${SIMDJSON_BUILD_STATIC} + -DSIMDJSON_DEVELOPMENT_CHECKS=OFF + -DSIMDJSON_VERBOSE_LOGGING=OFF + ${FEATURE_OPTIONS} +) + +vcpkg_cmake_install() + +vcpkg_copy_pdbs() + +vcpkg_cmake_config_fixup(CONFIG_PATH "lib/cmake/${PORT}") + +vcpkg_fixup_pkgconfig() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share") + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/dev/vcpkg/ports/simdjson/vcpkg.json b/dev/vcpkg/ports/simdjson/vcpkg.json new file mode 100644 index 000000000000..6e46382e42f8 --- /dev/null +++ b/dev/vcpkg/ports/simdjson/vcpkg.json @@ -0,0 +1,37 @@ +{ + "name": "simdjson", + "version": "3.9.3", + "description": "An extremely fast JSON library that can parse gigabytes of JSON per second", + "homepage": "https://simdjson.org/", + "license": "Apache-2.0", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "deprecated", + "exceptions", + "threads", + "utf8-validation" + ], + "features": { + "deprecated": { + "description": "Enable deprecated APIs" + }, + "exceptions": { + "description": "Enable exception-throwing interface" + }, + "threads": { + "description": "Link with thread support" + }, + "utf8-validation": { + "description": "Enable UTF-8 validation" + } + } +} diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index 4593c86e7d51..b7dcb059c082 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -117,7 +117,6 @@ }, "overrides": [ { "name": "fmt", "version": "10.1.1" }, - { "name": "xsimd", "version": "10.0.0" }, - { "name": "simdjson", "version": "3.2.0" } + { "name": "xsimd", "version": "10.0.0" } ] } diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index c13b49f30598..5d9eba904480 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -151,7 +151,7 @@ function compile { fi echo "NUM_THREADS_OPTS: $NUM_THREADS_OPTS" - export simdjson_SOURCE=BUNDLED + export simdjson_SOURCE=AUTO if [ $ARCH == 'x86_64' ]; then make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' ]]; then From 016b9e7d87295c7176929a32e17dfffb01262a9e Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:05:01 +0800 Subject: [PATCH 196/402] [VL] Daily Update Velox Version (2024_06_03) (#5956) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index ce25b20d43b8..e1133e88c717 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_05_31 +VELOX_BRANCH=2024_06_03 VELOX_HOME="" #Set on run gluten on HDFS From a76c92e82f75250cd834f51ee88f82a7664c6562 Mon Sep 17 00:00:00 2001 From: lgbo Date: Mon, 3 Jun 2024 16:09:41 +0800 Subject: [PATCH 197/402] [GLUTEN-5668][CH] Support mixed conditions in shuffle hash join (#5735) * support inequal join * fixed bugs in CI * fixed performance issue in broadcast join * broadcast join build changed --- .../clickhouse/CHSparkPlanExecApi.scala | 14 +- .../execution/CHHashJoinExecTransformer.scala | 14 +- .../CHSortMergeJoinExecTransformer.scala | 8 +- .../gluten/utils/CHJoinValidateUtil.scala | 94 ++-- ...kHouseTPCDSParquetGraceHashJoinSuite.scala | 153 +----- .../GlutenClickHouseTPCDSParquetSuite.scala | 69 +-- ...enClickHouseTPCHSaltNullParquetSuite.scala | 18 + .../Join/BroadCastJoinBuilder.cpp | 3 +- .../Join/StorageJoinFromReadBuffer.cpp | 129 ++++-- .../Join/StorageJoinFromReadBuffer.h | 29 +- cpp-ch/local-engine/Parser/JoinRelParser.cpp | 434 +++++++++++++----- cpp-ch/local-engine/Parser/JoinRelParser.h | 24 +- .../Parser/SerializedPlanParser.cpp | 2 +- 13 files changed, 522 insertions(+), 469 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 1403c8261df8..bdbdfed0d0d0 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -27,7 +27,7 @@ import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPreOverri import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, WindowFunctionNode} -import org.apache.gluten.utils.CHJoinValidateUtil +import org.apache.gluten.utils.{CHJoinValidateUtil, UnknownJoinStrategy} import org.apache.gluten.vectorized.CHColumnarBatchSerializer import org.apache.spark.{ShuffleDependency, SparkException} @@ -694,15 +694,19 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { } /** - * Define whether the join operator is fallback because of the join operator is not supported by - * backend + * This is only used to control whether transform smj into shj or not at present. We always prefer + * shj. */ override def joinFallback( - JoinType: JoinType, + joinType: JoinType, leftOutputSet: AttributeSet, rightOutputSet: AttributeSet, condition: Option[Expression]): Boolean = { - CHJoinValidateUtil.shouldFallback(JoinType, leftOutputSet, rightOutputSet, condition) + CHJoinValidateUtil.shouldFallback( + UnknownJoinStrategy(joinType), + leftOutputSet, + rightOutputSet, + condition) } /** Generate window function node */ diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala index c3ab89df5bb1..6004f7f861bf 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala @@ -18,7 +18,7 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.clickhouse.CHIteratorApi import org.apache.gluten.extension.ValidationResult -import org.apache.gluten.utils.CHJoinValidateUtil +import org.apache.gluten.utils.{BroadcastHashJoinStrategy, CHJoinValidateUtil, ShuffleHashJoinStrategy} import org.apache.spark.{broadcast, SparkContext} import org.apache.spark.rdd.RDD @@ -55,7 +55,11 @@ case class CHShuffledHashJoinExecTransformer( override protected def doValidateInternal(): ValidationResult = { val shouldFallback = - CHJoinValidateUtil.shouldFallback(joinType, left.outputSet, right.outputSet, condition) + CHJoinValidateUtil.shouldFallback( + ShuffleHashJoinStrategy(joinType), + left.outputSet, + right.outputSet, + condition) if (shouldFallback) { return ValidationResult.notOk("ch join validate fail") } @@ -107,7 +111,11 @@ case class CHBroadcastHashJoinExecTransformer( override protected def doValidateInternal(): ValidationResult = { val shouldFallback = - CHJoinValidateUtil.shouldFallback(joinType, left.outputSet, right.outputSet, condition) + CHJoinValidateUtil.shouldFallback( + BroadcastHashJoinStrategy(joinType), + left.outputSet, + right.outputSet, + condition) if (shouldFallback) { return ValidationResult.notOk("ch join validate fail") diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHSortMergeJoinExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHSortMergeJoinExecTransformer.scala index a5ac5f65840d..e2b586551739 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHSortMergeJoinExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHSortMergeJoinExecTransformer.scala @@ -17,7 +17,7 @@ package org.apache.gluten.execution import org.apache.gluten.extension.ValidationResult -import org.apache.gluten.utils.CHJoinValidateUtil +import org.apache.gluten.utils.{CHJoinValidateUtil, SortMergeJoinStrategy} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans._ @@ -44,7 +44,11 @@ case class CHSortMergeJoinExecTransformer( override protected def doValidateInternal(): ValidationResult = { val shouldFallback = - CHJoinValidateUtil.shouldFallback(joinType, left.outputSet, right.outputSet, condition, true) + CHJoinValidateUtil.shouldFallback( + SortMergeJoinStrategy(joinType), + left.outputSet, + right.outputSet, + condition) if (shouldFallback) { return ValidationResult.notOk("ch join validate fail") } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHJoinValidateUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHJoinValidateUtil.scala index 06b2445af6e1..dae8e6e073a1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHJoinValidateUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHJoinValidateUtil.scala @@ -17,9 +17,17 @@ package org.apache.gluten.utils import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.expressions.{AttributeSet, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, LessThan, LessThanOrEqual, Not, Or} +import org.apache.spark.sql.catalyst.expressions.{AttributeSet, Expression} import org.apache.spark.sql.catalyst.plans.JoinType +trait JoinStrategy { + val joinType: JoinType +} +case class UnknownJoinStrategy(joinType: JoinType) extends JoinStrategy {} +case class ShuffleHashJoinStrategy(joinType: JoinType) extends JoinStrategy {} +case class BroadcastHashJoinStrategy(joinType: JoinType) extends JoinStrategy {} +case class SortMergeJoinStrategy(joinType: JoinType) extends JoinStrategy {} + /** * The logic here is that if it is not an equi-join spark will create BNLJ, which will fallback, if * it is an equi-join, spark will create BroadcastHashJoin or ShuffleHashJoin, for these join types, @@ -34,78 +42,40 @@ object CHJoinValidateUtil extends Logging { def hasTwoTableColumn( leftOutputSet: AttributeSet, rightOutputSet: AttributeSet, - l: Expression, - r: Expression): Boolean = { - val allReferences = l.references ++ r.references + expr: Expression): Boolean = { + val allReferences = expr.references !(allReferences.subsetOf(leftOutputSet) || allReferences.subsetOf(rightOutputSet)) } def shouldFallback( - joinType: JoinType, + joinStrategy: JoinStrategy, leftOutputSet: AttributeSet, rightOutputSet: AttributeSet, - condition: Option[Expression], - isSMJ: Boolean = false): Boolean = { + condition: Option[Expression]): Boolean = { var shouldFallback = false + val joinType = joinStrategy.joinType if (joinType.toString.contains("ExistenceJoin")) { return true } - if (joinType.sql.equals("INNER")) { - return shouldFallback - } - if (isSMJ) { - if ( - joinType.sql.contains("SEMI") - || joinType.sql.contains("ANTI") - ) { - return true + if (joinType.sql.contains("INNER")) { + shouldFallback = false; + } else if ( + condition.isDefined && hasTwoTableColumn(leftOutputSet, rightOutputSet, condition.get) + ) { + shouldFallback = joinStrategy match { + case BroadcastHashJoinStrategy(joinTy) => + joinTy.sql.contains("SEMI") || joinTy.sql.contains("ANTI") + case SortMergeJoinStrategy(_) => true + case ShuffleHashJoinStrategy(joinTy) => + joinTy.sql.contains("SEMI") || joinTy.sql.contains("ANTI") + case UnknownJoinStrategy(joinTy) => + joinTy.sql.contains("SEMI") || joinTy.sql.contains("ANTI") } - } - if (condition.isDefined) { - condition.get.transform { - case Or(l, r) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - Or(l, r) - case Not(EqualTo(l, r)) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - Not(EqualTo(l, r)) - case LessThan(l, r) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - LessThan(l, r) - case LessThanOrEqual(l, r) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - LessThanOrEqual(l, r) - case GreaterThan(l, r) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - GreaterThan(l, r) - case GreaterThanOrEqual(l, r) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - GreaterThanOrEqual(l, r) - case In(l, r) => - r.foreach( - e => { - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, e)) { - shouldFallback = true - } - }) - In(l, r) - case EqualTo(l, r) => - if (hasTwoTableColumn(leftOutputSet, rightOutputSet, l, r)) { - shouldFallback = true - } - EqualTo(l, r) + } else { + shouldFallback = joinStrategy match { + case SortMergeJoinStrategy(joinTy) => + joinTy.sql.contains("SEMI") || joinTy.sql.contains("ANTI") + case _ => false } } shouldFallback diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala index 0b7ad9a6d8ac..04ccda29b1ae 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetGraceHashJoinSuite.scala @@ -16,13 +16,9 @@ */ package org.apache.gluten.execution -import org.apache.gluten.test.FallbackUtil - import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Not} -import org.apache.spark.sql.execution._ +import org.apache.spark.sql.catalyst.expressions.DynamicPruningExpression import org.apache.spark.sql.execution.exchange.ReusedExchangeExec -import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec class GlutenClickHouseTPCDSParquetGraceHashJoinSuite extends GlutenClickHouseTPCDSAbstractSuite { @@ -39,105 +35,11 @@ class GlutenClickHouseTPCDSParquetGraceHashJoinSuite extends GlutenClickHouseTPC .set("spark.sql.autoBroadcastJoinThreshold", "10MB") .set("spark.memory.offHeap.size", "8g") .set("spark.gluten.sql.columnar.backend.ch.runtime_settings.join_algorithm", "grace_hash") - .set("spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_in_join", "3145728") + .set("spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_in_join", "314572800") } executeTPCDSTest(false); - test( - "test fallback operations not supported by ch backend " + - "in CHHashJoinExecTransformer && CHBroadcastHashJoinExecTransformer") { - val testSql = - """ - | SELECT i_brand_id AS brand_id, i_brand AS brand, i_manufact_id, i_manufact, - | sum(ss_ext_sales_price) AS ext_price - | FROM date_dim - | LEFT JOIN store_sales ON d_date_sk = ss_sold_date_sk - | LEFT JOIN item ON ss_item_sk = i_item_sk AND i_manager_id = 7 - | LEFT JOIN customer ON ss_customer_sk = c_customer_sk - | LEFT JOIN customer_address ON c_current_addr_sk = ca_address_sk - | LEFT JOIN store ON ss_store_sk = s_store_sk AND substr(ca_zip,1,5) <> substr(s_zip,1,5) - | WHERE d_moy = 11 - | AND d_year = 1999 - | GROUP BY i_brand_id, i_brand, i_manufact_id, i_manufact - | ORDER BY ext_price DESC, i_brand, i_brand_id, i_manufact_id, i_manufact - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - val operateWithCondition = df.queryExecution.executedPlan.collect { - case f: BroadcastHashJoinExec if f.condition.get.isInstanceOf[Not] => f - } - assert( - operateWithCondition(0).left - .asInstanceOf[InputAdapter] - .child - .isInstanceOf[CHColumnarToRowExec]) - } - - test("test fallbackutils") { - val testSql = - """ - | SELECT i_brand_id AS brand_id, i_brand AS brand, i_manufact_id, i_manufact, - | sum(ss_ext_sales_price) AS ext_price - | FROM date_dim - | LEFT JOIN store_sales ON d_date_sk = ss_sold_date_sk - | LEFT JOIN item ON ss_item_sk = i_item_sk AND i_manager_id = 7 - | LEFT JOIN customer ON ss_customer_sk = c_customer_sk - | LEFT JOIN customer_address ON c_current_addr_sk = ca_address_sk - | LEFT JOIN store ON ss_store_sk = s_store_sk AND substr(ca_zip,1,5) <> substr(s_zip,1,5) - | WHERE d_moy = 11 - | AND d_year = 1999 - | GROUP BY i_brand_id, i_brand, i_manufact_id, i_manufact - | ORDER BY ext_price DESC, i_brand, i_brand_id, i_manufact_id, i_manufact - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - assert(FallbackUtil.hasFallback(df.queryExecution.executedPlan)) - } - - test("Gluten-4458: test clickhouse not support join with IN condition") { - val testSql = - """ - | SELECT * - | FROM date_dim t1 - | LEFT JOIN date_dim t2 ON t1.d_date_sk = t2.d_date_sk - | AND datediff(t1.d_day_name, t2.d_day_name) IN (1, 3) - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - assert(FallbackUtil.hasFallback(df.queryExecution.executedPlan)) - } - - test("Gluten-4458: test join with Equal computing two table in one side") { - val testSql = - """ - | SELECT * - | FROM date_dim t1 - | LEFT JOIN date_dim t2 ON t1.d_date_sk = t2.d_date_sk AND t1.d_year - t2.d_year = 1 - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - assert(FallbackUtil.hasFallback(df.queryExecution.executedPlan)) - } - - test("Gluten-4458: test inner join can support join with IN condition") { - val testSql = - """ - | SELECT * - | FROM date_dim t1 - | INNER JOIN date_dim t2 ON t1.d_date_sk = t2.d_date_sk - | AND datediff(t1.d_day_name, t2.d_day_name) IN (1, 3) - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - assert(!FallbackUtil.hasFallback(df.queryExecution.executedPlan)) - } - test("Gluten-1235: Fix missing reading from the broadcasted value when executing DPP") { val testSql = """ @@ -198,55 +100,4 @@ class GlutenClickHouseTPCDSParquetGraceHashJoinSuite extends GlutenClickHouseTPC } } } - - test("TPCDS Q21 with non-separated scan rdd") { - withSQLConf(("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false")) { - runTPCDSQuery("q21") { - df => - val foundDynamicPruningExpr = df.queryExecution.executedPlan.find { - case f: FileSourceScanExecTransformer => - f.partitionFilters.exists { - case _: DynamicPruningExpression => true - case _ => false - } - case _ => false - } - assert(foundDynamicPruningExpr.nonEmpty == true) - - val reuseExchange = df.queryExecution.executedPlan.find { - case r: ReusedExchangeExec => true - case _ => false - } - assert(reuseExchange.nonEmpty == true) - } - } - } - - test("Gluten-4452: Fix get wrong hash table when multi joins in a task") { - val testSql = - """ - | SELECT ws_item_sk, ws_sold_date_sk, ws_ship_date_sk, - | t3.d_date_id as sold_date_id, t2.d_date_id as ship_date_id - | FROM ( - | SELECT ws_item_sk, ws_sold_date_sk, ws_ship_date_sk, t1.d_date_id - | FROM web_sales - | LEFT JOIN - | (SELECT d_date_id, d_date_sk from date_dim GROUP BY d_date_id, d_date_sk) t1 - | ON ws_sold_date_sk == t1.d_date_sk) t3 - | INNER JOIN - | (SELECT d_date_id, d_date_sk from date_dim GROUP BY d_date_id, d_date_sk) t2 - | ON ws_ship_date_sk == t2.d_date_sk - | LIMIT 100; - |""".stripMargin - compareResultsAgainstVanillaSpark( - testSql, - true, - df => { - val foundBroadcastHashJoinExpr = df.queryExecution.executedPlan.collect { - case f: CHBroadcastHashJoinExecTransformer => f - } - assert(foundBroadcastHashJoinExpr.size == 2) - } - ) - } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala index a63e47888cb9..e9c27437b44a 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSParquetSuite.scala @@ -16,13 +16,10 @@ */ package org.apache.gluten.execution -import org.apache.gluten.test.FallbackUtil - import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Not} +import org.apache.spark.sql.catalyst.expressions.DynamicPruningExpression import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.exchange.ReusedExchangeExec -import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, SortMergeJoinExec} // Some sqls' line length exceeds 100 // scalastyle:off line.size.limit @@ -121,38 +118,7 @@ class GlutenClickHouseTPCDSParquetSuite extends GlutenClickHouseTPCDSAbstractSui assert(result(0).getLong(0) == 73049) } - test( - "test fallback operations not supported by ch backend " + - "in CHHashJoinExecTransformer && CHBroadcastHashJoinExecTransformer") { - val testSql = - """ - |SELECT i_brand_id AS brand_id, i_brand AS brand, i_manufact_id, i_manufact, - | sum(ss_ext_sales_price) AS ext_price - | FROM date_dim - | LEFT JOIN store_sales ON d_date_sk = ss_sold_date_sk - | LEFT JOIN item ON ss_item_sk = i_item_sk AND i_manager_id = 7 - | LEFT JOIN customer ON ss_customer_sk = c_customer_sk - | LEFT JOIN customer_address ON c_current_addr_sk = ca_address_sk - | LEFT JOIN store ON ss_store_sk = s_store_sk AND substr(ca_zip,1,5) <> substr(s_zip,1,5) - | WHERE d_moy = 11 - | AND d_year = 1999 - | GROUP BY i_brand_id, i_brand, i_manufact_id, i_manufact - | ORDER BY ext_price DESC, i_brand, i_brand_id, i_manufact_id, i_manufact - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - val operateWithCondition = df.queryExecution.executedPlan.collect { - case f: BroadcastHashJoinExec if f.condition.get.isInstanceOf[Not] => f - } - assert( - operateWithCondition(0).left - .asInstanceOf[InputAdapter] - .child - .isInstanceOf[CHColumnarToRowExec]) - } - - test("test fallbackutils") { + test("Test join with mixed condition 1") { val testSql = """ |SELECT i_brand_id AS brand_id, i_brand AS brand, i_manufact_id, i_manufact, @@ -169,36 +135,7 @@ class GlutenClickHouseTPCDSParquetSuite extends GlutenClickHouseTPCDSAbstractSui | ORDER BY ext_price DESC, i_brand, i_brand_id, i_manufact_id, i_manufact | LIMIT 100; |""".stripMargin - - val df = spark.sql(testSql) - assert(FallbackUtil.hasFallback(df.queryExecution.executedPlan)) - } - - test( - "Test avoid forceShuffledHashJoin when the join condition" + - " does not supported by the backend") { - val testSql = - """ - |SELECT /*+ merge(date_dim)*/ i_brand_id AS brand_id, i_brand AS brand, i_manufact_id, i_manufact, - | sum(ss_ext_sales_price) AS ext_price - | FROM date_dim - | LEFT JOIN store_sales ON d_date_sk == ss_sold_date_sk AND (d_date_sk = 213232 OR ss_sold_date_sk = 3232) - | LEFT JOIN item ON ss_item_sk = i_item_sk AND i_manager_id = 7 - | LEFT JOIN customer ON ss_customer_sk = c_customer_sk - | LEFT JOIN customer_address ON c_current_addr_sk = ca_address_sk - | LEFT JOIN store ON ss_store_sk = s_store_sk AND substr(ca_zip,1,5) <> substr(s_zip,1,5) - | WHERE d_moy = 11 - | AND d_year = 1999 - | GROUP BY i_brand_id, i_brand, i_manufact_id, i_manufact - | ORDER BY ext_price DESC, i_brand, i_brand_id, i_manufact_id, i_manufact - | LIMIT 100; - |""".stripMargin - - val df = spark.sql(testSql) - val sortMergeJoinExec = df.queryExecution.executedPlan.collect { - case s: SortMergeJoinExec => s - } - assert(sortMergeJoinExec.nonEmpty) + compareResultsAgainstVanillaSpark(testSql, true, _ => {}) } test("Gluten-1235: Fix missing reading from the broadcasted value when executing DPP") { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 748bd5a7f7f6..038b170df491 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2563,5 +2563,23 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) spark.sql("drop table test_tbl_5896") } + + test("Inequal join support") { + withSQLConf(("spark.sql.autoBroadcastJoinThreshold", "-1")) { + spark.sql("create table ineq_join_t1 (key bigint, value bigint) using parquet"); + spark.sql("create table ineq_join_t2 (key bigint, value bigint) using parquet"); + spark.sql("insert into ineq_join_t1 values(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)"); + spark.sql("insert into ineq_join_t2 values(2, 2), (2, 1), (3, 3), (4, 6), (5, 3)"); + val sql = + """ + | select t1.key, t1.value, t2.key, t2.value from ineq_join_t1 as t1 + | left join ineq_join_t2 as t2 + | on t1.key = t2.key and t1.value > t2.value + |""".stripMargin + compareResultsAgainstVanillaSpark(sql, true, { _ => }) + spark.sql("drop table ineq_join_t1") + spark.sql("drop table ineq_join_t2") + } + } } // scalastyle:on line.size.limit diff --git a/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp b/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp index d90951241166..f1b3ac2fbd9c 100644 --- a/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp +++ b/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp @@ -103,7 +103,8 @@ std::shared_ptr buildJoin( row_count, key_names, true, - std::make_shared(SizeLimits(), true, kind, strictness, key_names), + kind, + strictness, columns_description, ConstraintsDescription(), key, diff --git a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp index 6d0021adbf40..f0aec6af686d 100644 --- a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp +++ b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp @@ -15,7 +15,9 @@ * limitations under the License. */ #include "StorageJoinFromReadBuffer.h" +#include +#include #include #include #include @@ -23,6 +25,9 @@ #include #include +#include +#include + namespace DB { class HashJoin; @@ -40,25 +45,23 @@ extern const int DEADLOCK_AVOIDED; using namespace DB; -void restore(DB::ReadBuffer & in, IJoin & join, const Block & sample_block) -{ - local_engine::NativeReader block_stream(in); - ProfileInfo info; - while (Block block = block_stream.read()) - { - auto final_block = sample_block.cloneWithColumns(block.mutateColumns()); - info.update(final_block); - join.addBlockToJoin(final_block, true); - } -} +constexpr auto RIHGT_COLUMN_PREFIX = "broadcast_right_"; DB::Block rightSampleBlock(bool use_nulls, const StorageInMemoryMetadata & storage_metadata_, JoinKind kind) { + DB::ColumnsWithTypeAndName new_cols; DB::Block block = storage_metadata_.getSampleBlock(); - if (use_nulls && isLeftOrFull(kind)) - for (auto & col : block) - DB::JoinCommon::convertColumnToNullable(col); - return block; + for (const auto & col : block) + { + // Add a prefix to avoid column name conflicts with left table. + new_cols.emplace_back(col.column, col.type, RIHGT_COLUMN_PREFIX + col.name); + if (use_nulls && isLeftOrFull(kind)) + { + auto & new_col = new_cols.back(); + DB::JoinCommon::convertColumnToNullable(new_col); + } + } + return DB::Block(new_cols); } namespace local_engine @@ -67,46 +70,88 @@ namespace local_engine StorageJoinFromReadBuffer::StorageJoinFromReadBuffer( DB::ReadBuffer & in, size_t row_count_, - const Names & key_names, - bool use_nulls, - std::shared_ptr table_join, + const Names & key_names_, + bool use_nulls_, + DB::JoinKind kind, + DB::JoinStrictness strictness, const ColumnsDescription & columns, const ConstraintsDescription & constraints, const String & comment, - const bool overwrite) - : key_names_(key_names), use_nulls_(use_nulls) + const bool overwrite_) + : key_names({}), use_nulls(use_nulls_), row_count(row_count_), overwrite(overwrite_) { - storage_metadata_.setColumns(columns); - storage_metadata_.setConstraints(constraints); - storage_metadata_.setComment(comment); + storage_metadata.setColumns(columns); + storage_metadata.setConstraints(constraints); + storage_metadata.setComment(comment); - for (const auto & key : key_names) - if (!storage_metadata_.getColumns().hasPhysical(key)) + for (const auto & key : key_names_) + if (!storage_metadata.getColumns().hasPhysical(key)) throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "Key column ({}) does not exist in table declaration.", key); - right_sample_block_ = rightSampleBlock(use_nulls, storage_metadata_, table_join->kind()); - join_ = std::make_shared(table_join, right_sample_block_, overwrite, row_count_); - restore(in, *join_, storage_metadata_.getSampleBlock()); + for (const auto & name : key_names_) + key_names.push_back(RIHGT_COLUMN_PREFIX + name); + auto table_join = std::make_shared(SizeLimits(), true, kind, strictness, key_names); + right_sample_block = rightSampleBlock(use_nulls, storage_metadata, table_join->kind()); + buildJoin(in, right_sample_block, table_join); +} + +/// The column names may be different in two blocks. +/// and the nullability also could be different, with TPCDS-Q1 as an example. +static DB::ColumnWithTypeAndName convertColumnAsNecessary(const DB::ColumnWithTypeAndName & column, const DB::ColumnWithTypeAndName & sample_column) +{ + if (sample_column.type->equals(*column.type)) + return {column.column, column.type, sample_column.name}; + else if ( + sample_column.type->isNullable() && !column.type->isNullable() + && DB::removeNullable(sample_column.type)->equals(*column.type)) + { + auto nullable_column = column; + DB::JoinCommon::convertColumnToNullable(nullable_column); + return {nullable_column.column, sample_column.type, sample_column.name}; + } + else + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "Columns have different types. original:{} expected:{}", + column.dumpStructure(), + sample_column.dumpStructure()); +} + +void StorageJoinFromReadBuffer::buildJoin(DB::ReadBuffer & in, const Block header, std::shared_ptr analyzed_join) +{ + local_engine::NativeReader block_stream(in); + ProfileInfo info; + join = std::make_shared(analyzed_join, header, overwrite, row_count); + while (Block block = block_stream.read()) + { + DB::ColumnsWithTypeAndName columns; + for (size_t i = 0; i < block.columns(); ++i) + { + const auto & column = block.getByPosition(i); + columns.emplace_back(convertColumnAsNecessary(column, header.getByPosition(i))); + } + DB::Block final_block(columns); + info.update(final_block); + join->addBlockToJoin(final_block, true); + } } -DB::JoinPtr StorageJoinFromReadBuffer::getJoinLocked(std::shared_ptr analyzed_join, DB::ContextPtr /*context*/) const +/// The column names of 'rgiht_header' could be different from the ones in `input_blocks`, and we must +/// use 'right_header' to build the HashJoin. Otherwise, it will cause exceptions with name mismatches. +/// +/// In most cases, 'getJoinLocked' is called only once, and the input_blocks should not be too large. +/// This is will be OK. +DB::JoinPtr StorageJoinFromReadBuffer::getJoinLocked(std::shared_ptr analyzed_join, DB::ContextPtr /*context*/) { - if ((analyzed_join->forceNullableRight() && !use_nulls_) - || (!analyzed_join->forceNullableRight() && isLeftOrFull(analyzed_join->kind()) && use_nulls_)) + if ((analyzed_join->forceNullableRight() && !use_nulls) + || (!analyzed_join->forceNullableRight() && isLeftOrFull(analyzed_join->kind()) && use_nulls)) throw Exception( ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN, "Table {} needs the same join_use_nulls setting as present in LEFT or FULL JOIN", - storage_metadata_.comment); - - /// TODO: check key columns - - /// Set names qualifiers: table.column -> column - /// It's required because storage join stores non-qualified names - /// Qualifies will be added by join implementation (HashJoin) - analyzed_join->setRightKeys(key_names_); - - HashJoinPtr join_clone = std::make_shared(analyzed_join, right_sample_block_); - join_clone->reuseJoinedData(static_cast(*join_)); + storage_metadata.comment); + HashJoinPtr join_clone = std::make_shared(analyzed_join, right_sample_block); + /// reuseJoinedData will set the flag `HashJoin::from_storage_join` which is required by `FilledStep` + join_clone->reuseJoinedData(static_cast(*join)); return join_clone; } } diff --git a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h index 2e949fa87c67..af623c0cd717 100644 --- a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h +++ b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h @@ -23,6 +23,8 @@ namespace DB class TableJoin; class IJoin; using JoinPtr = std::shared_ptr; +class HashJoin; +class ReadBuffer; } namespace local_engine @@ -33,23 +35,32 @@ class StorageJoinFromReadBuffer public: StorageJoinFromReadBuffer( DB::ReadBuffer & in_, - size_t row_count_, + size_t row_count, const DB::Names & key_names_, bool use_nulls_, - std::shared_ptr table_join_, + DB::JoinKind kind, + DB::JoinStrictness strictness, const DB::ColumnsDescription & columns_, const DB::ConstraintsDescription & constraints_, const String & comment, bool overwrite_); - DB::JoinPtr getJoinLocked(std::shared_ptr analyzed_join, DB::ContextPtr context) const; - const DB::Block & getRightSampleBlock() const { return right_sample_block_; } + /// The columns' names in right_header may be different from the names in the ColumnsDescription + /// in the constructor. + /// This should be called once. + DB::JoinPtr getJoinLocked(std::shared_ptr analyzed_join, DB::ContextPtr context); + const DB::Block & getRightSampleBlock() const { return right_sample_block; } private: - DB::StorageInMemoryMetadata storage_metadata_; - const DB::Names key_names_; - bool use_nulls_; - DB::JoinPtr join_; - DB::Block right_sample_block_; + DB::StorageInMemoryMetadata storage_metadata; + DB::Names key_names; + bool use_nulls; + size_t row_count; + bool overwrite; + DB::Block right_sample_block; + std::shared_ptr join = nullptr; + + void readAllBlocksFromInput(DB::ReadBuffer & in); + void buildJoin(DB::ReadBuffer & in, const DB::Block header, std::shared_ptr analyzed_join); }; } diff --git a/cpp-ch/local-engine/Parser/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/JoinRelParser.cpp index 8f7f35d5ef23..937e449b0825 100644 --- a/cpp-ch/local-engine/Parser/JoinRelParser.cpp +++ b/cpp-ch/local-engine/Parser/JoinRelParser.cpp @@ -31,6 +31,10 @@ #include #include +#include +#include + + namespace DB { namespace ErrorCodes @@ -179,40 +183,167 @@ DB::QueryPlanPtr JoinRelParser::parseOp(const substrait::Rel & rel, std::list JoinRelParser::extractTableSidesFromExpression(const substrait::Expression & expr, const DB::Block & left_header, const DB::Block & right_header) +{ + std::unordered_set table_sides; + if (expr.has_scalar_function()) + { + for (const auto & arg : expr.scalar_function().arguments()) + { + auto table_sides_from_arg = extractTableSidesFromExpression(arg.value(), left_header, right_header); + table_sides.insert(table_sides_from_arg.begin(), table_sides_from_arg.end()); + } + } + else if (expr.has_selection() && expr.selection().has_direct_reference() && expr.selection().direct_reference().has_struct_field()) + { + auto pos = expr.selection().direct_reference().struct_field().field(); + if (pos < left_header.columns()) + { + table_sides.insert(DB::JoinTableSide::Left); + } + else + { + table_sides.insert(DB::JoinTableSide::Right); + } + } + else if (expr.has_singular_or_list()) + { + auto child_table_sides = extractTableSidesFromExpression(expr.singular_or_list().value(), left_header, right_header); + table_sides.insert(child_table_sides.begin(), child_table_sides.end()); + for (const auto & option : expr.singular_or_list().options()) + { + child_table_sides = extractTableSidesFromExpression(option, left_header, right_header); + table_sides.insert(child_table_sides.begin(), child_table_sides.end()); + } + } + else if (expr.has_cast()) + { + auto child_table_sides = extractTableSidesFromExpression(expr.cast().input(), left_header, right_header); + table_sides.insert(child_table_sides.begin(), child_table_sides.end()); + } + else if (expr.has_if_then()) + { + for (const auto & if_child : expr.if_then().ifs()) + { + auto child_table_sides = extractTableSidesFromExpression(if_child.if_(), left_header, right_header); + table_sides.insert(child_table_sides.begin(), child_table_sides.end()); + child_table_sides = extractTableSidesFromExpression(if_child.then(), left_header, right_header); + table_sides.insert(child_table_sides.begin(), child_table_sides.end()); + } + auto child_table_sides = extractTableSidesFromExpression(expr.if_then().else_(), left_header, right_header); + table_sides.insert(child_table_sides.begin(), child_table_sides.end()); + } + else if (expr.has_literal()) + { + // nothing + } + else + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Illegal expression '{}'", expr.DebugString()); + } + return table_sides; +} + + +void JoinRelParser::renamePlanColumns(DB::QueryPlan & left, DB::QueryPlan & right, const StorageJoinFromReadBuffer & storage_join) +{ + /// To support mixed join conditions, we must make sure that the column names in the right be the same as + /// storage_join's right sample block. + ActionsDAGPtr project = ActionsDAG::makeConvertingActions( + right.getCurrentDataStream().header.getColumnsWithTypeAndName(), + storage_join.getRightSampleBlock().getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Position); + + if (project) + { + QueryPlanStepPtr project_step = std::make_unique(right.getCurrentDataStream(), project); + project_step->setStepDescription("Rename Broadcast Table Name"); + steps.emplace_back(project_step.get()); + right.addStep(std::move(project_step)); + } + + /// If the columns name in right table is duplicated with left table, we need to rename the left table's columns, + /// avoid the columns name in the right table be changed in `addConvertStep`. + /// This could happen in tpc-ds q44. + DB::ColumnsWithTypeAndName new_left_cols; + const auto & right_header = right.getCurrentDataStream().header; + auto left_prefix = getUniqueName("left"); + for (const auto & col : left.getCurrentDataStream().header) + { + if (right_header.has(col.name)) + { + new_left_cols.emplace_back(col.column, col.type, left_prefix + col.name); + } + else + { + new_left_cols.emplace_back(col.column, col.type, col.name); + } + } + project = ActionsDAG::makeConvertingActions( + left.getCurrentDataStream().header.getColumnsWithTypeAndName(), + new_left_cols, + ActionsDAG::MatchColumnsMode::Position); + + if (project) + { + QueryPlanStepPtr project_step = std::make_unique(left.getCurrentDataStream(), project); + project_step->setStepDescription("Rename Left Table Name for broadcast join"); + steps.emplace_back(project_step.get()); + left.addStep(std::move(project_step)); + } +} + DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::QueryPlanPtr left, DB::QueryPlanPtr right) { auto join_opt_info = parseJoinOptimizationInfo(join); auto storage_join = join_opt_info.is_broadcast ? BroadCastJoinBuilder::getJoin(join_opt_info.storage_join_key) : nullptr; - if (storage_join) { - ActionsDAGPtr project = ActionsDAG::makeConvertingActions( - right->getCurrentDataStream().header.getColumnsWithTypeAndName(), - storage_join->getRightSampleBlock().getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Position); + renamePlanColumns(*left, *right, *storage_join); + } + + auto table_join = createDefaultTableJoin(join.type()); + DB::Block right_header_before_convert_step = right->getCurrentDataStream().header; + addConvertStep(*table_join, *left, *right); - if (project) + // Add a check to find error easily. + if (storage_join) + { + if(!blocksHaveEqualStructure(right_header_before_convert_step, right->getCurrentDataStream().header)) { - QueryPlanStepPtr project_step = std::make_unique(right->getCurrentDataStream(), project); - project_step->setStepDescription("Rename Broadcast Table Name"); - steps.emplace_back(project_step.get()); - right->addStep(std::move(project_step)); + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "For broadcast join, we must not change the columns name in the right table.\nleft header:{},\nright header: {} -> {}", + left->getCurrentDataStream().header.dumpNames(), + right_header_before_convert_step.dumpNames(), + right->getCurrentDataStream().header.dumpNames()); } } - auto table_join = createDefaultTableJoin(join.type()); - addConvertStep(*table_join, *left, *right); Names after_join_names; auto left_names = left->getCurrentDataStream().header.getNames(); after_join_names.insert(after_join_names.end(), left_names.begin(), left_names.end()); auto right_name = table_join->columnsFromJoinedTable().getNames(); after_join_names.insert(after_join_names.end(), right_name.begin(), right_name.end()); - bool add_filter_step = tryAddPushDownFilter(*table_join, join, *left, *right, table_join->columnsFromJoinedTable(), after_join_names); + + auto left_header = left->getCurrentDataStream().header; + auto right_header = right->getCurrentDataStream().header; QueryPlanPtr query_plan; + + /// Support only one join clause. + table_join->addDisjunct(); + /// some examples to explain when the post_join_filter is not empty + /// - on t1.key = t2.key and t1.v1 > 1 and t2.v1 > 1, 't1.v1> 1' is in the post filter. but 't2.v1 > 1' + /// will be pushed down into right table by spark and is not in the post filter. 't1.key = t2.key ' is + /// in JoinRel::expression. + /// - on t1.key = t2. key and t1.v1 > t2.v2, 't1.v1 > t2.v2' is in the post filter. + collectJoinKeys(*table_join, join, left_header, right_header); + if (storage_join) { + + applyJoinFilter(*table_join, join, *left, *right, true); auto broadcast_hash_join = storage_join->getJoinLocked(table_join, context); + QueryPlanStepPtr join_step = std::make_unique(left->getCurrentDataStream(), broadcast_hash_join, 8192); join_step->setStepDescription("STORAGE_JOIN"); @@ -224,6 +355,18 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::Q } else if (join_opt_info.is_smj) { + bool need_post_filter = !applyJoinFilter(*table_join, join, *left, *right, false); + + /// If applyJoinFilter returns false, it means there are mixed conditions in the post_join_filter. + /// It should be a inner join. + /// TODO: make smj support mixed conditions + if (need_post_filter && table_join->kind() != DB::JoinKind::Inner) + { + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "Sort merge join doesn't support mixed join conditions, except inner join."); + } + JoinPtr smj_join = std::make_shared(table_join, right->getCurrentDataStream().header.cloneEmpty(), -1); MultiEnum join_algorithm = context->getSettingsRef().join_algorithm; QueryPlanStepPtr join_step @@ -237,12 +380,14 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::Q query_plan = std::make_unique(); query_plan->unitePlans(std::move(join_step), {std::move(plans)}); + if (need_post_filter) + addPostFilter(*query_plan, join); } else { - /// TODO: make grace hash join be the default hash join algorithm. - /// - /// Following is some configuration for grace hash join. + applyJoinFilter(*table_join, join, *left, *right, true); + + /// Following is some configurations for grace hash join. /// - spark.gluten.sql.columnar.backend.ch.runtime_settings.join_algorithm=grace_hash. This will /// enable grace hash join. /// - spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_in_join=3145728. This setup @@ -278,28 +423,15 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::Q } reorderJoinOutput(*query_plan, after_join_names); - if (add_filter_step) - { - addPostFilter(*query_plan, join); - } return query_plan; } void JoinRelParser::addConvertStep(TableJoin & table_join, DB::QueryPlan & left, DB::QueryPlan & right) { - - /// After https://github.com/ClickHouse/ClickHouse/pull/61216, We will failed at tryPushDownFilter() in filterPushDown.cpp - /// Here is a workaround, refer to chooseJoinAlgorithm() in PlannerJoins.cpp, it always call TableJoin::setRename to - /// create aliases for columns in the right table - /// By using right table header name sets, so TableJoin::deduplicateAndQualifyColumnNames can do same thing as chooseJoinAlgorithm() - /// - /// Affected UT fixed bh this workaround: - /// GlutenClickHouseTPCHParquetRFSuite:TPCH Q17, Q19, Q20, Q21 + /// If the columns name in right table is duplicated with left table, we need to rename the right table's columns. NameSet left_columns_set; - for (const auto & col : right.getCurrentDataStream().header.getNames()) - { + for (const auto & col : left.getCurrentDataStream().header.getNames()) left_columns_set.emplace(col); - } table_join.setColumnsFromJoinedTable( right.getCurrentDataStream().header.getNamesAndTypesList(), left_columns_set, getUniqueName("right") + "."); @@ -360,117 +492,179 @@ void JoinRelParser::addConvertStep(TableJoin & table_join, DB::QueryPlan & left, } } -void JoinRelParser::addPostFilter(DB::QueryPlan & query_plan, const substrait::JoinRel & join) +/// Join keys are collected from substrait::JoinRel::expression() which only contains the equal join conditions. +void JoinRelParser::collectJoinKeys( + TableJoin & table_join, const substrait::JoinRel & join_rel, const DB::Block & left_header, const DB::Block & right_header) { - std::string filter_name; - auto actions_dag = std::make_shared(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); - if (!join.post_join_filter().has_scalar_function()) + if (!join_rel.has_expression()) + return; + const auto & expr = join_rel.expression(); + auto & join_clause = table_join.getClauses().back(); + std::list expressions_stack; + expressions_stack.push_back(&expr); + while (!expressions_stack.empty()) { - // It may be singular_or_list - auto * in_node = getPlanParser()->parseExpression(actions_dag, join.post_join_filter()); - filter_name = in_node->result_name; - } - else - { - getPlanParser()->parseFunction(query_plan.getCurrentDataStream().header, join.post_join_filter(), filter_name, actions_dag, true); + /// Must handle the expressions in DF order. It matters in sort merge join. + const auto * current_expr = expressions_stack.back(); + expressions_stack.pop_back(); + if (!current_expr->has_scalar_function()) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Function expression is expected"); + auto function_name = parseFunctionName(current_expr->scalar_function().function_reference(), current_expr->scalar_function()); + if (!function_name) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Invalid function expression"); + if (*function_name == "equals") + { + String left_key, right_key; + size_t left_pos = 0, right_pos = 0; + for (const auto & arg : current_expr->scalar_function().arguments()) + { + if (!arg.value().has_selection() || !arg.value().selection().has_direct_reference() + || !arg.value().selection().direct_reference().has_struct_field()) + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "A column reference is expected"); + } + auto col_pos_ref = arg.value().selection().direct_reference().struct_field().field(); + if (col_pos_ref < left_header.columns()) + { + left_pos = col_pos_ref; + left_key = left_header.getByPosition(col_pos_ref).name; + } + else + { + right_pos = col_pos_ref - left_header.columns(); + right_key = right_header.getByPosition(col_pos_ref - left_header.columns()).name; + } + } + if (left_key.empty() || right_key.empty()) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Invalid key equal join condition"); + join_clause.addKey(left_key, right_key, false); + } + else if (*function_name == "and") + { + expressions_stack.push_back(¤t_expr->scalar_function().arguments().at(1).value()); + expressions_stack.push_back(¤t_expr->scalar_function().arguments().at(0).value()); + } + else + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknow function: {}", *function_name); + } } - auto filter_step = std::make_unique(query_plan.getCurrentDataStream(), actions_dag, filter_name, true); - filter_step->setStepDescription("Post Join Filter"); - steps.emplace_back(filter_step.get()); - query_plan.addStep(std::move(filter_step)); } -bool JoinRelParser::tryAddPushDownFilter( - TableJoin & table_join, - const substrait::JoinRel & join, - DB::QueryPlan & left, - DB::QueryPlan & right, - const NamesAndTypesList & alias_right, - const Names & names) +bool JoinRelParser::applyJoinFilter( + DB::TableJoin & table_join, const substrait::JoinRel & join_rel, DB::QueryPlan & left, DB::QueryPlan & right, bool allow_mixed_condition) { - try + if (!join_rel.has_post_join_filter()) + return true; + const auto & expr = join_rel.post_join_filter(); + + const auto & left_header = left.getCurrentDataStream().header; + const auto & right_header = right.getCurrentDataStream().header; + ColumnsWithTypeAndName mixed_columns; + std::unordered_set added_column_name; + for (const auto & col : left_header.getColumnsWithTypeAndName()) + { + mixed_columns.emplace_back(col); + added_column_name.insert(col.name); + } + for (const auto & col : right_header.getColumnsWithTypeAndName()) { - ASTParser astParser(context, function_mapping, getPlanParser()); - ASTs args; + const auto & renamed_col_name = table_join.renamedRightColumnNameWithAlias(col.name); + if (added_column_name.find(col.name) != added_column_name.end()) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Right column's name conflict with left column: {}", col.name); + mixed_columns.emplace_back(col); + added_column_name.insert(col.name); + } + DB::Block mixed_header(mixed_columns); - if (join.has_expression()) - { - args.emplace_back(astParser.parseToAST(names, join.expression())); - } + auto table_sides = extractTableSidesFromExpression(expr, left_header, right_header); - if (join.has_post_join_filter()) + auto get_input_expressions = [](const DB::Block & header) + { + std::vector exprs; + for (size_t i = 0; i < header.columns(); ++i) { - args.emplace_back(astParser.parseToAST(names, join.post_join_filter())); + substrait::Expression expr; + expr.mutable_selection()->mutable_direct_reference()->mutable_struct_field()->set_field(i); + exprs.emplace_back(expr); } - - if (args.empty()) - return false; - - ASTPtr ast = args.size() == 1 ? args.back() : makeASTFunction("and", args); - - bool is_asof = (table_join.strictness() == JoinStrictness::Asof); - - Aliases aliases; - DatabaseAndTableWithAlias left_table_name; - DatabaseAndTableWithAlias right_table_name; - TableWithColumnNamesAndTypes left_table(left_table_name, left.getCurrentDataStream().header.getNamesAndTypesList()); - TableWithColumnNamesAndTypes right_table(right_table_name, alias_right); - - CollectJoinOnKeysVisitor::Data data{table_join, left_table, right_table, aliases, is_asof}; - if (auto * or_func = ast->as(); or_func && or_func->name == "or") + return exprs; + }; + + /// If the columns in the expression are all from one table, use analyzer_left_filter_condition_column_name + /// and analyzer_left_filter_condition_column_name to filt the join result data. It requires to build the filter + /// column at first. + /// If the columns in the expression are from both tables, use mixed_join_expression to filt the join result data. + /// the filter columns will be built inner the join step. + if (table_sides.size() == 1) + { + auto table_side = *table_sides.begin(); + if (table_side == DB::JoinTableSide::Left) { - for (auto & disjunct : or_func->arguments->children) - { - table_join.addDisjunct(); - CollectJoinOnKeysVisitor(data).visit(disjunct); - } - assert(table_join.getClauses().size() == or_func->arguments->children.size()); + auto input_exprs = get_input_expressions(left_header); + input_exprs.push_back(expr); + auto actions_dag = expressionsToActionsDAG(input_exprs, left_header); + table_join.getClauses().back().analyzer_left_filter_condition_column_name = actions_dag->getOutputs().back()->result_name; + QueryPlanStepPtr before_join_step = std::make_unique(left.getCurrentDataStream(), actions_dag); + before_join_step->setStepDescription("Before JOIN LEFT"); + steps.emplace_back(before_join_step.get()); + left.addStep(std::move(before_join_step)); } else { - table_join.addDisjunct(); - CollectJoinOnKeysVisitor(data).visit(ast); - assert(table_join.oneDisjunct()); - } - - if (join.has_post_join_filter()) - { - auto left_keys = table_join.leftKeysList(); - auto right_keys = table_join.rightKeysList(); - if (!left_keys->children.empty()) + /// since the field reference in expr is the index of left_header ++ right_header, so we use + /// mixed_header to build the actions_dag + auto input_exprs = get_input_expressions(mixed_header); + input_exprs.push_back(expr); + auto actions_dag = expressionsToActionsDAG(input_exprs, mixed_header); + + /// clear unused columns in actions_dag + for (const auto & col : left_header.getColumnsWithTypeAndName()) { - auto actions = astParser.convertToActions(left.getCurrentDataStream().header.getNamesAndTypesList(), left_keys); - QueryPlanStepPtr before_join_step = std::make_unique(left.getCurrentDataStream(), actions); - before_join_step->setStepDescription("Before JOIN LEFT"); - steps.emplace_back(before_join_step.get()); - left.addStep(std::move(before_join_step)); + actions_dag->removeUnusedResult(col.name); } + actions_dag->removeUnusedActions(); - if (!right_keys->children.empty()) - { - auto actions = astParser.convertToActions(right.getCurrentDataStream().header.getNamesAndTypesList(), right_keys); - QueryPlanStepPtr before_join_step = std::make_unique(right.getCurrentDataStream(), actions); - before_join_step->setStepDescription("Before JOIN RIGHT"); - steps.emplace_back(before_join_step.get()); - right.addStep(std::move(before_join_step)); - } + table_join.getClauses().back().analyzer_right_filter_condition_column_name = actions_dag->getOutputs().back()->result_name; + QueryPlanStepPtr before_join_step = std::make_unique(right.getCurrentDataStream(), actions_dag); + before_join_step->setStepDescription("Before JOIN RIGHT"); + steps.emplace_back(before_join_step.get()); + right.addStep(std::move(before_join_step)); } } - // if ch does not support the join type or join conditions, it will throw an exception like 'not support'. - catch (Poco::Exception & e) + else if (table_sides.size() == 2) { - // CH not support join condition has 'or' and has different table in each side. - // But in inner join, we could execute join condition after join. so we have add filter step - if (e.code() == ErrorCodes::INVALID_JOIN_ON_EXPRESSION && table_join.kind() == DB::JoinKind::Inner) - { - return true; - } - else - { - throw; - } + if (!allow_mixed_condition) + return false; + auto mixed_join_expressions_actions = expressionsToActionsDAG({expr}, mixed_header); + table_join.getMixedJoinExpression() + = std::make_shared(mixed_join_expressions_actions, ExpressionActionsSettings::fromContext(context)); } - return false; + else + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Not any table column is used in the join condition.\n{}", join_rel.DebugString()); + } + return true; +} + +void JoinRelParser::addPostFilter(DB::QueryPlan & query_plan, const substrait::JoinRel & join) +{ + std::string filter_name; + auto actions_dag = std::make_shared(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); + if (!join.post_join_filter().has_scalar_function()) + { + // It may be singular_or_list + auto * in_node = getPlanParser()->parseExpression(actions_dag, join.post_join_filter()); + filter_name = in_node->result_name; + } + else + { + getPlanParser()->parseFunction(query_plan.getCurrentDataStream().header, join.post_join_filter(), filter_name, actions_dag, true); + } + auto filter_step = std::make_unique(query_plan.getCurrentDataStream(), actions_dag, filter_name, true); + filter_step->setStepDescription("Post Join Filter"); + steps.emplace_back(filter_step.get()); + query_plan.addStep(std::move(filter_step)); } void registerJoinRelParser(RelParserFactory & factory) diff --git a/cpp-ch/local-engine/Parser/JoinRelParser.h b/cpp-ch/local-engine/Parser/JoinRelParser.h index 445b7e683300..c423f43908e7 100644 --- a/cpp-ch/local-engine/Parser/JoinRelParser.h +++ b/cpp-ch/local-engine/Parser/JoinRelParser.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -28,6 +29,8 @@ class TableJoin; namespace local_engine { +class StorageJoinFromReadBuffer; + std::pair getJoinKindAndStrictness(substrait::JoinRel_JoinType join_type); class JoinRelParser : public RelParser @@ -50,15 +53,22 @@ class JoinRelParser : public RelParser DB::QueryPlanPtr parseJoin(const substrait::JoinRel & join, DB::QueryPlanPtr left, DB::QueryPlanPtr right); + void renamePlanColumns(DB::QueryPlan & left, DB::QueryPlan & right, const StorageJoinFromReadBuffer & storage_join); void addConvertStep(TableJoin & table_join, DB::QueryPlan & left, DB::QueryPlan & right); - bool tryAddPushDownFilter( - TableJoin & table_join, - const substrait::JoinRel & join, - DB::QueryPlan & left, - DB::QueryPlan & right, - const NamesAndTypesList & alias_right, - const Names & names); + void collectJoinKeys( + TableJoin & table_join, const substrait::JoinRel & join_rel, const DB::Block & left_header, const DB::Block & right_header); + + bool applyJoinFilter( + DB::TableJoin & table_join, + const substrait::JoinRel & join_rel, + DB::QueryPlan & left_plan, + DB::QueryPlan & right_plan, + bool allow_mixed_condition); + void addPostFilter(DB::QueryPlan & plan, const substrait::JoinRel & join); + + static std::unordered_set extractTableSidesFromExpression( + const substrait::Expression & expr, const DB::Block & left_header, const DB::Block & right_header); }; } diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index a26f78699dc8..b0d3bbeca962 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -214,7 +214,7 @@ std::shared_ptr SerializedPlanParser::expressionsToActionsDAG( } } } - else if (expr.has_cast() || expr.has_if_then() || expr.has_literal()) + else if (expr.has_cast() || expr.has_if_then() || expr.has_literal() || expr.has_singular_or_list()) { const auto * node = parseExpression(actions_dag, expr); actions_dag->addOrReplaceInOutputs(*node); From ac52ba01a687cc219bf0fe5ff68153101678e2bc Mon Sep 17 00:00:00 2001 From: Chang chen Date: Tue, 4 Jun 2024 09:47:09 +0800 Subject: [PATCH 198/402] [GLUTEN-3582][CH] Support FLBAType and BOOLEAN (#5962) [CH] Support FLBAType and BOOLEAN --- .../GlutenParquetColumnIndexSuite.scala | 34 +++++- .../Storages/Parquet/ColumnIndexFilter.cpp | 32 ++++-- .../Storages/Parquet/ParquetConverter.h | 108 ++++++++++++++++-- .../tests/gtest_parquet_columnindex.cpp | 101 +++++++++++++--- 4 files changed, 233 insertions(+), 42 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala index bc23728524d9..05ed7ed6b842 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala @@ -25,7 +25,12 @@ import org.apache.spark.sql.DataFrame import org.apache.spark.sql.gluten.test.GlutenSQLTestUtils import org.apache.spark.sql.internal.SQLConf -case class ParquetData(parquetDir: String, filter: String, scanOutput: Long) +case class ParquetData( + column: String, + parquetDir: String, + filter: String, + scanOutput: Long, + title: Option[String] = None) class GlutenParquetColumnIndexSuite extends GlutenClickHouseWholeStageTransformerSuite @@ -39,20 +44,41 @@ class GlutenParquetColumnIndexSuite // both gluten and vanilla spark dataframe private val parquetData = Seq( ParquetData( + "count(*)", "index/tpch/20003", "`27` <> '1-URGENT' and `9` >= '1995-01-01' and `9` < '1996-01-01' ", 140000), ParquetData( + "count(*)", "index/tpch/upper_case", "c_comment = '! requests wake. (...)ructions. furiousl'", - 12853) + 12853), + ParquetData( + "*", + "index/pageindex/query102", + "`198` = 'Crafts' or `198` = 'Computers' or `198`= 'a' or `198`= ''", + 45), + ParquetData( + "count(*)", + "index/pageindex/query102", + "`100001` < 30000 and `100001` > 1000.004", + 45, + Some("push down Decimal filter")), + ParquetData( + "count(*)", + "index/pageindex/query102", + "`100001` in (30000, 1000.004, 45000, 2323445, 4235423.6, 4546677.245, 56677.5)", + 45, + Some("push down Decimal filter In") + ), + ParquetData("count(*)", "index/pageindex/query05", "`142` = true", 9896) ) parquetData.foreach { data => - test(s"${data.parquetDir}") { + test(data.title.getOrElse(data.parquetDir)) { val parquetDir = s"$testPath/${data.parquetDir}" - val sql1 = s"""|select count(*) from $fileFormat.`$parquetDir` + val sql1 = s"""|select ${data.column} from $fileFormat.`$parquetDir` |where ${data.filter} |""".stripMargin compareResultsAgainstVanillaSpark( diff --git a/cpp-ch/local-engine/Storages/Parquet/ColumnIndexFilter.cpp b/cpp-ch/local-engine/Storages/Parquet/ColumnIndexFilter.cpp index 0d3b07e4786a..817de7f27ef8 100644 --- a/cpp-ch/local-engine/Storages/Parquet/ColumnIndexFilter.cpp +++ b/cpp-ch/local-engine/Storages/Parquet/ColumnIndexFilter.cpp @@ -547,7 +547,8 @@ PageIndexs TypedColumnIndexImpl::notEq(const DB::Field & value) co } // Merging value filtering with pages containing nulls - auto real_value{parquetCast(value)}; + ToParquet to_parquet; + auto real_value{to_parquet.as(value, *descr_)}; TypedComparator typed_comparator{real_value, *column_index_, *comparator_}; auto pages = ORDER::notEq(typed_comparator); const std::set matchingIndexes(pages.begin(), pages.end()); @@ -573,7 +574,8 @@ PageIndexs TypedColumnIndexImpl::eq(const DB::Field & value) const return {PageIndexsBuilder::ALL_PAGES}; } } - auto real_value = parquetCast(value); + ToParquet to_parquet; + auto real_value{to_parquet.as(value, *descr_)}; TypedComparator typed_comparator{real_value, *column_index_, *comparator_}; return ORDER::eq(typed_comparator); } @@ -581,7 +583,8 @@ PageIndexs TypedColumnIndexImpl::eq(const DB::Field & value) const template ORDER> PageIndexs TypedColumnIndexImpl::gt(const DB::Field & value) const { - auto real_value{parquetCast(value)}; + ToParquet to_parquet; + auto real_value{to_parquet.as(value, *descr_)}; TypedComparator typed_comparator{real_value, *column_index_, *comparator_}; return ORDER::gt(typed_comparator); } @@ -589,7 +592,8 @@ PageIndexs TypedColumnIndexImpl::gt(const DB::Field & value) const template ORDER> PageIndexs TypedColumnIndexImpl::gtEg(const DB::Field & value) const { - auto real_value{parquetCast(value)}; + ToParquet to_parquet; + auto real_value{to_parquet.as(value, *descr_)}; TypedComparator typed_comparator{real_value, *column_index_, *comparator_}; return ORDER::gtEq(typed_comparator); } @@ -597,7 +601,8 @@ PageIndexs TypedColumnIndexImpl::gtEg(const DB::Field & value) con template ORDER> PageIndexs TypedColumnIndexImpl::lt(const DB::Field & value) const { - auto real_value{parquetCast(value)}; + ToParquet to_parquet; + auto real_value{to_parquet.as(value, *descr_)}; TypedComparator typed_comparator{real_value, *column_index_, *comparator_}; return ORDER::lt(typed_comparator); } @@ -605,7 +610,8 @@ PageIndexs TypedColumnIndexImpl::lt(const DB::Field & value) const template ORDER> PageIndexs TypedColumnIndexImpl::ltEg(const DB::Field & value) const { - auto real_value{parquetCast(value)}; + ToParquet to_parquet; + auto real_value{to_parquet.as(value, *descr_)}; TypedComparator typed_comparator{real_value, *column_index_, *comparator_}; return ORDER::ltEq(typed_comparator); } @@ -615,7 +621,7 @@ PageIndexs TypedColumnIndexImpl::in(const DB::ColumnPtr & column) { /// TDDO: handle null /// - std::shared_ptr> converter = ParquetConverter::Make(column); + std::shared_ptr> converter = ParquetConverter::Make(column, *descr_); const auto * value = converter->getBatch(0, column->size()); T min, max; std::tie(min, max) = comparator_->GetMinMax(value, column->size()); @@ -659,7 +665,8 @@ ColumnIndexPtr internalMakeColumnIndex( switch (physical_type) { case parquet::Type::BOOLEAN: - break; + return std::make_unique>( + descr, dynamic_pointer_cast(column_index), offset_index); case parquet::Type::INT32: return std::make_unique>( descr, dynamic_pointer_cast(column_index), offset_index); @@ -669,20 +676,21 @@ ColumnIndexPtr internalMakeColumnIndex( case parquet::Type::INT96: break; case parquet::Type::FLOAT: - break; + return std::make_unique>( + descr, dynamic_pointer_cast(column_index), offset_index); case parquet::Type::DOUBLE: return std::make_unique>( descr, dynamic_pointer_cast(column_index), offset_index); - break; case parquet::Type::BYTE_ARRAY: return std::make_unique>( descr, dynamic_pointer_cast(column_index), offset_index); case parquet::Type::FIXED_LEN_BYTE_ARRAY: - break; + return std::make_unique>( + descr, dynamic_pointer_cast(column_index), offset_index); case parquet::Type::UNDEFINED: break; } - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unsupported physical type {}", physical_type); + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unsupported physical type {}", TypeToString(physical_type)); } ColumnIndexPtr ColumnIndex::create( diff --git a/cpp-ch/local-engine/Storages/Parquet/ParquetConverter.h b/cpp-ch/local-engine/Storages/Parquet/ParquetConverter.h index ac7b2479abe2..89e83e668aeb 100644 --- a/cpp-ch/local-engine/Storages/Parquet/ParquetConverter.h +++ b/cpp-ch/local-engine/Storages/Parquet/ParquetConverter.h @@ -15,33 +15,72 @@ * limitations under the License. */ #pragma once +#include #include +#include +#include #include #include #include +namespace DB::ErrorCodes +{ +extern const int LOGICAL_ERROR; +} + namespace local_engine { + template -auto parquetCast(const DB::Field & value) -> typename PhysicalType::c_type +struct ToParquet { using T = typename PhysicalType::c_type; - if constexpr (std::is_same_v) - return static_cast(value.get()); - else if constexpr (std::is_same_v) + T as(const DB::Field & value, const parquet::ColumnDescriptor &) + { + if constexpr (std::is_same_v) + return static_cast(value.get()); + // parquet::BooleanType, parquet::Int64Type, parquet::FloatType, parquet::DoubleType + return value.get(); // FLOAT, DOUBLE, INT64 + } +}; + +template <> +struct ToParquet +{ + using T = parquet::ByteArray; + T as(const DB::Field & value, const parquet::ColumnDescriptor &) { assert(value.getType() == DB::Field::Types::String); const std::string & s = value.get(); const auto * const ptr = reinterpret_cast(s.data()); return parquet::ByteArray(static_cast(s.size()), ptr); } - else if constexpr (std::is_same_v) +}; + +template <> +struct ToParquet +{ + uint8_t buf[256]; + using T = parquet::FixedLenByteArray; + T as(const DB::Field & value, const parquet::ColumnDescriptor & descriptor) { - abort(); + if (value.getType() != DB::Field::Types::Decimal128) + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, "Field type '{}' for FIXED_LEN_BYTE_ARRAY is not supported", value.getTypeName()); + static_assert(sizeof(Int128) <= sizeof(buf)); + if (descriptor.type_length() > sizeof(Int128)) + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "descriptor.type_length() = {} , which is > {}, e.g. sizeof(Int128)", + descriptor.type_length(), + sizeof(Int128)); + Int128 val = value.get>().getValue(); + std::reverse(reinterpret_cast(&val), reinterpret_cast(&val) + sizeof(val)); + const int offset = sizeof(Int128) - descriptor.type_length(); + memcpy(buf, reinterpret_cast(&val) + offset, descriptor.type_length()); + return parquet::FixedLenByteArray(buf); } - else - return value.get(); // FLOAT, DOUBLE, INT64 -} +}; // Int32 Int64 Float Double template @@ -100,6 +139,42 @@ struct ConverterString } }; +/// Like ConverterNumberAsFixedString, but converts to big-endian. Because that's the byte order +/// Parquet uses for decimal types and literally nothing else, for some reason. +template +struct ConverterDecimal +{ + const parquet::ColumnDescriptor & descriptor; + const DB::ColumnDecimal & column; + DB::PODArray data_buf; + DB::PODArray ptr_buf; + + explicit ConverterDecimal(const DB::ColumnPtr & c, const parquet::ColumnDescriptor & desc) + : descriptor(desc), column(assert_cast &>(*c)) + { + if (descriptor.type_length() > sizeof(T)) + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "descriptor.type_length() = {} , which is > {}, e.g. sizeof(T)", + descriptor.type_length(), + sizeof(T)); + } + + const parquet::FixedLenByteArray * getBatch(size_t offset, size_t count) + { + data_buf.resize(count * sizeof(T)); + ptr_buf.resize(count); + memcpy(data_buf.data(), reinterpret_cast(column.getData().data() + offset), count * sizeof(T)); + const size_t offset_in_buf = sizeof(Int128) - descriptor.type_length(); + ; + for (size_t i = 0; i < count; ++i) + { + std::reverse(data_buf.data() + i * sizeof(T), data_buf.data() + (i + 1) * sizeof(T)); + ptr_buf[i].ptr = data_buf.data() + i * sizeof(T) + offset_in_buf; + } + return ptr_buf.data(); + } +}; class BaseConverter { @@ -115,7 +190,7 @@ class ParquetConverter : public BaseConverter public: virtual const T * getBatch(size_t offset, size_t count) = 0; - static std::shared_ptr> Make(const DB::ColumnPtr & c); + static std::shared_ptr> Make(const DB::ColumnPtr & c, const parquet::ColumnDescriptor & desc); }; template @@ -134,7 +209,7 @@ class ParquetConverterImpl final : public ParquetConverter template -std::shared_ptr> ParquetConverter::Make(const DB::ColumnPtr & c) +std::shared_ptr> ParquetConverter::Make(const DB::ColumnPtr & c, const parquet::ColumnDescriptor & desc) { std::shared_ptr result; @@ -204,6 +279,17 @@ std::shared_ptr> ParquetConverter::Make(const DB: break; } break; + case parquet::Type::FIXED_LEN_BYTE_ARRAY: + switch (c->getDataType()) + { + case TypeIndex::Decimal128: + result = std::make_shared>>( + ConverterDecimal(c, desc)); + break; + default: + break; + } + break; default: break; } diff --git a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp index ba09b21b266e..ea3bd41e4384 100644 --- a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp +++ b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp @@ -15,12 +15,14 @@ * limitations under the License. */ +#include + + #include "config.h" #if USE_PARQUET #include #include #include -#include #include #include #include @@ -59,6 +61,9 @@ class PrimitiveNodeBuilder parquet::Repetition::type repetition_ = parquet::Repetition::UNDEFINED; parquet::ConvertedType::type converted_type_ = parquet::ConvertedType::NONE; parquet::Type::type physical_type_ = parquet::Type::UNDEFINED; + int length_ = -1; + int precision_ = -1; + int scale_ = -1; public: PrimitiveNodeBuilder & as(parquet::ConvertedType::type converted_type) @@ -67,13 +72,25 @@ class PrimitiveNodeBuilder return *this; } + PrimitiveNodeBuilder & with_length(int length) + { + length_ = length; + return *this; + } + PrimitiveNodeBuilder & asDecimal(int precision, int scale) + { + converted_type_ = parquet::ConvertedType::DECIMAL; + precision_ = precision; + scale_ = scale; + return *this; + } parquet::schema::NodePtr named(const std::string & name) const { assert(!name.empty()); if (physical_type_ == parquet::Type::UNDEFINED) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsupported physical type"); return parquet::schema::PrimitiveNode::Make( - name, repetition_, physical_type_, converted_type_, /*length=*/-1, /*precision=*/-1, /*scale=*/-1, /*field_id*/ -1); + name, repetition_, physical_type_, converted_type_, length_, precision_, scale_, /*field_id*/ -1); } parquet::ColumnDescriptor descriptor(const std::string & name) const { return {named(name), /*max_definition_level=*/1, 0}; } static PrimitiveNodeBuilder optional(parquet::Type::type physical_type) @@ -483,13 +500,22 @@ using ParquetValue = std::variant< parquet::DoubleType::c_type, parquet::ByteArrayType::c_type>; -ParquetValue to(const DB::Field & value, const parquet::ColumnDescriptor & desc) +template +void doComapre( + const parquet::ColumnDescriptor & descriptor, const DB::Field & value, const std::function & compare) +{ + local_engine::ToParquet to_parquet; + compare({to_parquet.as(value, descriptor)}); +} + +void with_actual(const DB::Field & value, const parquet::ColumnDescriptor & desc, const std::function & compare) { using namespace local_engine; switch (desc.physical_type()) { case parquet::Type::BOOLEAN: - break; + doComapre(desc, value, compare); + return; case parquet::Type::INT32: { switch (desc.converted_type()) { @@ -500,7 +526,8 @@ ParquetValue to(const DB::Field & value, const parquet::ColumnDescriptor & desc) case parquet::ConvertedType::INT_16: case parquet::ConvertedType::INT_32: case parquet::ConvertedType::NONE: - return {parquetCast(value)}; + doComapre(desc, value, compare); + return; default: break; } @@ -512,35 +539,81 @@ ParquetValue to(const DB::Field & value, const parquet::ColumnDescriptor & desc) case parquet::ConvertedType::INT_64: case parquet::ConvertedType::UINT_64: case parquet::ConvertedType::NONE: - return {parquetCast(value)}; + doComapre(desc, value, compare); + return; default: break; } break; case parquet::Type::INT96: + // doComapre(desc, value, compare); break; case parquet::Type::FLOAT: - return {value.get()}; + doComapre(desc, value, compare); + return; case parquet::Type::DOUBLE: - return {value.get()}; - break; + doComapre(desc, value, compare); + return; case parquet::Type::BYTE_ARRAY: switch (desc.converted_type()) { case parquet::ConvertedType::UTF8: - return parquetCast(value); + doComapre(desc, value, compare); + return; default: break; } break; case parquet::Type::FIXED_LEN_BYTE_ARRAY: + // doComapre(desc, value, compare); break; case parquet::Type::UNDEFINED: break; } - abort(); + ASSERT_TRUE(false) << "Unsupported physical type: [" << TypeToString(desc.physical_type()) << "] with logical type: [" + << desc.logical_type()->ToString() << "] with converted type: [" << ConvertedTypeToString(desc.converted_type()) + << "]"; } +// for gtest +namespace parquet +{ +void PrintTo(const ByteArray & val, std::ostream * os) +{ + *os << '[' << std::hex; + + for (size_t i = 0; i < val.len; ++i) + { + *os << std::setw(2) << std::setfill('0') << static_cast(val.ptr[i]); + if (i != val.len - 1) + *os << ", "; + } + *os << ']'; +} +} +TEST(ColumnIndex, DecimalField) +{ + // we can't define `operator==` for parquet::FLBAType + Field value = DecimalField(Int128(300000000), 4); + local_engine::ToParquet to_parquet; + const parquet::ColumnDescriptor desc + = PNB::optional(parquet::Type::FIXED_LEN_BYTE_ARRAY).asDecimal(38, 4).with_length(13).descriptor("column1"); + uint8_t expected_a[13]{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x11, 0xE1, 0xA3, 0x0}; + const parquet::ByteArray expected{13, expected_a}; + const parquet::ByteArray actual{13, to_parquet.as(value, desc).ptr}; + ASSERT_EQ(actual, expected); + + + /// Eexception test + Field unsupport = DecimalField(Int256(300000000), 4); + EXPECT_THROW(to_parquet.as(unsupport, desc), DB::Exception); + + const parquet::ColumnDescriptor error + = PNB::optional(parquet::Type::FIXED_LEN_BYTE_ARRAY).asDecimal(38, 4).with_length(18).descriptor("column1"); + EXPECT_THROW(to_parquet.as(value, error), DB::Exception); +} + + TEST(ColumnIndex, Field) { std::string s_tmp = "hello world"; @@ -551,7 +624,6 @@ TEST(ColumnIndex, Field) parquet::ColumnDescriptor, //desc ParquetValue //expected value >; - using PNB = test_utils::PrimitiveNodeBuilder; const std::vector datas{ {"int32_UINT_8", static_cast(1), @@ -579,8 +651,7 @@ TEST(ColumnIndex, Field) const auto & value = std::get<1>(data); const auto & desc = std::get<2>(data); const auto & expected = std::get<3>(data); - const auto actual = to(value, desc); - ASSERT_EQ(actual, expected) << name; + with_actual(value, desc, [&](const ParquetValue & actual) { ASSERT_EQ(actual, expected) << name; }); }); const std::vector> primitive_fields{ @@ -612,7 +683,7 @@ struct ReadStatesParam ReadStatesParam() = default; ReadStatesParam(local_engine::RowRanges ranges, std::shared_ptr states) - : row_ranges(std::move(ranges)), read_states(std::move(states)){}; + : row_ranges(std::move(ranges)), read_states(std::move(states)) {}; local_engine::RowRanges row_ranges; std::shared_ptr read_states; From 20322ceb8b1aa70ad48020267b145e6b936d8cf4 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 4 Jun 2024 10:03:28 +0800 Subject: [PATCH 199/402] [VL][CI] Update mirror for Centos8 (#5970) --- .github/workflows/velox_docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index bbd713d99077..39ee14557451 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -549,6 +549,10 @@ jobs: ./cpp/build/releases/ /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }} + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | From 785958e0d3693cf7529237119a6c918d8639a833 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 4 Jun 2024 10:42:49 +0800 Subject: [PATCH 200/402] [VL] Gluten-it: Add option --scan-partitions (#5958) --- .../apache/gluten/integration/BaseMixin.java | 12 +++++------ .../org/apache/gluten/integration/Suite.scala | 21 ++++++++++--------- .../clickbench/ClickBenchSuite.scala | 4 ++-- .../gluten/integration/ds/TpcdsSuite.scala | 4 ++-- .../gluten/integration/h/TpchSuite.scala | 4 ++-- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java index 41d244871b75..dc1691e50021 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java @@ -72,11 +72,11 @@ public class BaseMixin { @CommandLine.Option(names = {"--disable-wscg"}, description = "Disable Spark SQL whole stage code generation", defaultValue = "false") private boolean disableWscg; - @CommandLine.Option(names = {"--shuffle-partitions"}, description = "Generate data with partitions", defaultValue = "100") + @CommandLine.Option(names = {"--shuffle-partitions"}, description = "Shuffle partition number", defaultValue = "100") private int shufflePartitions; - @CommandLine.Option(names = {"--min-scan-partitions"}, description = "Use minimum number of partitions to read data", defaultValue = "false") - private boolean minimumScanPartitions; + @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value", defaultValue = "100") + private int scanPartitions; @CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark config entries applying to generated Spark session. E.g. --extra-conf=k1=v1 --extra-conf=k2=v2") private Map extraSparkConf = Collections.emptyMap(); @@ -131,19 +131,19 @@ public Integer runActions(Action[] actions) { suite = new TpchSuite(runModeEnumeration.getSparkMasterUrl(), actions, testConf, baselineConf, extraSparkConfScala, level, errorOnMemLeak, enableUi, enableHsUi, hsUiPort, offHeapSize, disableAqe, disableBhj, - disableWscg, shufflePartitions, minimumScanPartitions); + disableWscg, shufflePartitions, scanPartitions); break; case "ds": suite = new TpcdsSuite(runModeEnumeration.getSparkMasterUrl(), actions, testConf, baselineConf, extraSparkConfScala, level, errorOnMemLeak, enableUi, enableHsUi, hsUiPort, offHeapSize, disableAqe, disableBhj, - disableWscg, shufflePartitions, minimumScanPartitions); + disableWscg, shufflePartitions, scanPartitions); break; case "clickbench": suite = new ClickBenchSuite(runModeEnumeration.getSparkMasterUrl(), actions, testConf, baselineConf, extraSparkConfScala, level, errorOnMemLeak, enableUi, enableHsUi, hsUiPort, offHeapSize, disableAqe, disableBhj, - disableWscg, shufflePartitions, minimumScanPartitions); + disableWscg, shufflePartitions, scanPartitions); break; default: throw new IllegalArgumentException("TPC benchmark type not found: " + benchmarkType); diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala index 9e31e11713a5..bb5cb1889125 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala @@ -43,7 +43,7 @@ abstract class Suite( private val disableBhj: Boolean, private val disableWscg: Boolean, private val shufflePartitions: Int, - private val minimumScanPartitions: Boolean) { + private val scanPartitions: Int) { resetLogLevel() @@ -103,15 +103,16 @@ abstract class Suite( sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.codegen.wholeStage", "false") } - if (minimumScanPartitions) { - sessionSwitcher - .defaultConf() - .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}") - sessionSwitcher - .defaultConf() - .setWarningOnOverriding("spark.sql.files.openCostInBytes", s"${ByteUnit.PiB.toBytes(1L)}") - sessionSwitcher.defaultConf().setWarningOnOverriding("spark.default.parallelism", "1") - } + // Scan partition number. + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}") + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0") + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}") extraSparkConf.toStream.foreach { kv => sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala index deffdb7e556a..04a34d332b61 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchSuite.scala @@ -43,7 +43,7 @@ class ClickBenchSuite( val disableBhj: Boolean, val disableWscg: Boolean, val shufflePartitions: Int, - val minimumScanPartitions: Boolean) + val scanPartitions: Int) extends Suite( masterUrl, actions, @@ -60,7 +60,7 @@ class ClickBenchSuite( disableBhj, disableWscg, shufflePartitions, - minimumScanPartitions) { + scanPartitions) { import ClickBenchSuite._ override protected def historyWritePath(): String = HISTORY_WRITE_PATH diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala index 339e89d5be71..a4365afde38d 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/ds/TpcdsSuite.scala @@ -42,7 +42,7 @@ class TpcdsSuite( val disableBhj: Boolean, val disableWscg: Boolean, val shufflePartitions: Int, - val minimumScanPartitions: Boolean) + val scanPartitions: Int) extends Suite( masterUrl, actions, @@ -59,7 +59,7 @@ class TpcdsSuite( disableBhj, disableWscg, shufflePartitions, - minimumScanPartitions) { + scanPartitions) { override protected def historyWritePath(): String = HISTORY_WRITE_PATH diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala index 29c299beebf1..bdcac3bffe39 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/h/TpchSuite.scala @@ -38,7 +38,7 @@ class TpchSuite( val disableBhj: Boolean, val disableWscg: Boolean, val shufflePartitions: Int, - val minimumScanPartitions: Boolean) + val scanPartitions: Int) extends Suite( masterUrl, actions, @@ -55,7 +55,7 @@ class TpchSuite( disableBhj, disableWscg, shufflePartitions, - minimumScanPartitions) { + scanPartitions) { override protected def historyWritePath(): String = HISTORY_WRITE_PATH From 27135096ae60ac06b1cf28a57ba0c4a66ec43e31 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Tue, 4 Jun 2024 14:08:22 +0800 Subject: [PATCH 201/402] [VL] Remove reselect build side in ShuffledHashJoinExecTransformer (#5935) --- .../ShuffledHashJoinExecTransformer.scala | 79 +------------------ 1 file changed, 2 insertions(+), 77 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala index 002afea31624..d78e6c5b3e53 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/ShuffledHashJoinExecTransformer.scala @@ -18,10 +18,9 @@ package org.apache.gluten.execution import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} +import org.apache.spark.sql.catalyst.optimizer.BuildSide import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.execution.{FilterExec, SparkPlan} -import org.apache.spark.sql.execution.aggregate.BaseAggregateExec +import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.joins.BuildSideRelation import org.apache.spark.sql.vectorized.ColumnarBatch @@ -46,80 +45,6 @@ case class ShuffledHashJoinExecTransformer( right, isSkewJoin) { - // Used to specify the preferred build side in backend's real execution. - object PreferredBuildSide extends Serializable { - val LEFT = "left table" - val RIGHT = "right table" - val NON = "none" - } - - /** - * Returns whether the plan matches the condition to be preferred as build side. Currently, filter - * and aggregation are preferred. - * @param plan - * the left or right plan of join - * @return - * whether the plan matches the condition - */ - private def matchCondition(plan: SparkPlan): Boolean = - plan.isInstanceOf[FilterExecTransformerBase] || plan.isInstanceOf[FilterExec] || - plan.isInstanceOf[BaseAggregateExec] - - /** - * Returns whether a plan is preferred as the build side. If this plan or its children match the - * condition, it will be preferred. - * @param plan - * the left or right plan of join - * @return - * whether the plan is preferred as the build side - */ - private def isPreferred(plan: SparkPlan): Boolean = - matchCondition(plan) || plan.children.exists(child => matchCondition(child)) - - // Returns the preferred build side with the consideration of preferring condition. - private lazy val preferredBuildSide: String = - if ((isPreferred(left) && isPreferred(right)) || (!isPreferred(left) && !isPreferred(right))) { - PreferredBuildSide.NON - } else if (isPreferred(left)) { - PreferredBuildSide.LEFT - } else { - PreferredBuildSide.RIGHT - } - - /** - * Returns whether the build and stream table should be exchanged with consideration of build - * type, planned build side and the preferred build side. - */ - override lazy val needSwitchChildren: Boolean = hashJoinType match { - case LeftOuter | LeftSemi | ExistenceJoin(_) => - joinBuildSide match { - case BuildLeft => - // Exchange build and stream side when left side or none is preferred as the build side, - // and RightOuter or RightSemi wil be used. - !(preferredBuildSide == PreferredBuildSide.RIGHT) - case _ => - // Do not exchange build and stream side when right side or none is preferred - // as the build side, and LeftOuter or LeftSemi wil be used. - preferredBuildSide == PreferredBuildSide.LEFT - } - case RightOuter => - joinBuildSide match { - case BuildRight => - // Do not exchange build and stream side when right side or none is preferred - // as the build side, and RightOuter will be used. - preferredBuildSide == PreferredBuildSide.LEFT - case _ => - // Exchange build and stream side when left side or none is preferred as the build side, - // and LeftOuter will be used. - !(preferredBuildSide == PreferredBuildSide.RIGHT) - } - case _ => - joinBuildSide match { - case BuildLeft => true - case BuildRight => false - } - } - override protected lazy val substraitJoinType: JoinRel.JoinType = joinType match { case _: InnerLike => JoinRel.JoinType.JOIN_TYPE_INNER From 9a6526d29e3b1b72f10e861068552674f1549ab3 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 4 Jun 2024 15:19:34 +0800 Subject: [PATCH 202/402] [VL][CI] Follow-up: update centos-8 mirror list (#5972) --- .github/workflows/velox_docker.yml | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 39ee14557451..dc007c5760aa 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -607,6 +607,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -681,6 +685,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -732,6 +740,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -798,6 +810,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -849,6 +865,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -915,6 +935,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -966,6 +990,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y @@ -1031,6 +1059,10 @@ jobs: with: name: arrow-jars-centos-8-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Update mirror list + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y From 1cf7b34cfb346d204c252410c98df142297c85c6 Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Tue, 4 Jun 2024 16:39:58 +0800 Subject: [PATCH 203/402] [CH] Fix left and substring with length -1 (#5943) What changes were proposed in this pull request? Fix left and substring with length -1 expect empty string How was this patch tested? unit tests (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) --- ...enClickHouseTPCHSaltNullParquetSuite.scala | 47 +++++++++++++------ .../scalar_function_parser/substring.cpp | 4 +- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 038b170df491..d2752a0739c0 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -722,7 +722,8 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr } test("test literals") { - val query = """ + val query = + """ SELECT CAST(NULL AS BOOLEAN) AS boolean_literal, CAST(1 AS TINYINT) AS tinyint_literal, @@ -1321,9 +1322,10 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("create table test_1767 (id bigint, data map) using parquet") spark.sql("INSERT INTO test_1767 values(1, map('k', 'v'))") - val sql = """ - | select id from test_1767 lateral view - | posexplode(split(data['k'], ',')) tx as a, b""".stripMargin + val sql = + """ + | select id from test_1767 lateral view + | posexplode(split(data['k'], ',')) tx as a, b""".stripMargin runQueryAndCompare(sql)(checkGlutenOperatorMatch[CHGenerateExecTransformer]) spark.sql("drop table test_1767") @@ -2082,21 +2084,23 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr } test("GLUTEN-3149 convert Nan to int") { - val sql = """ - | select cast(a as Int) as n from( - | select cast(s as Float) as a from( - | select if(n_name='ALGERIA', 'nan', '1.0') as s from nation - | ))""".stripMargin + val sql = + """ + | select cast(a as Int) as n from( + | select cast(s as Float) as a from( + | select if(n_name='ALGERIA', 'nan', '1.0') as s from nation + | ))""".stripMargin compareResultsAgainstVanillaSpark(sql, true, { _ => }) } test("GLUTEN-3149 convert Inf to int") { - val sql = """ - | select n_regionkey, n is null, isnan(n), cast(n as int) from ( - | select n_regionkey, x, n_regionkey/(x) as n from ( - | select n_regionkey, cast(n_nationkey as float) as x from nation - | )t1 - | )t2""".stripMargin + val sql = + """ + | select n_regionkey, n is null, isnan(n), cast(n as int) from ( + | select n_regionkey, x, n_regionkey/(x) as n from ( + | select n_regionkey, cast(n_nationkey as float) as x from nation + | )t1 + | )t2""".stripMargin compareResultsAgainstVanillaSpark(sql, true, { _ => }) } @@ -2564,6 +2568,19 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("drop table test_tbl_5896") } + test("test left with len -1") { + val tbl_create_sql = + "create table test_left(col string) using parquet" + val tbl_insert_sql = + "insert into test_left values('test1'), ('test2')" + spark.sql(tbl_create_sql) + spark.sql(tbl_insert_sql) + compareResultsAgainstVanillaSpark("select left(col, -1) from test_left", true, { _ => }) + compareResultsAgainstVanillaSpark("select left(col, -2) from test_left", true, { _ => }) + compareResultsAgainstVanillaSpark("select substring(col, 0, -1) from test_left", true, { _ => }) + spark.sql("drop table test_left") + } + test("Inequal join support") { withSQLConf(("spark.sql.autoBroadcastJoinThreshold", "-1")) { spark.sql("create table ineq_join_t1 (key bigint, value bigint) using parquet"); diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/substring.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/substring.cpp index 64c97da80581..550e77344ddf 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/substring.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/substring.cpp @@ -59,7 +59,9 @@ class FunctionParserSubstring : public FunctionParser const auto * const_one_node = addColumnToActionsDAG(actions_dag, index_type, 1); const auto * equals_zero_node = toFunctionNode(actions_dag, "equals", {index_arg, const_zero_node}); const auto * if_node = toFunctionNode(actions_dag, "if", {equals_zero_node, const_one_node, index_arg}); - const auto * substring_func_node = toFunctionNode(actions_dag, "substringUTF8", {str_arg, if_node, length_arg}); + const auto * less_zero_node = toFunctionNode(actions_dag, "less", {length_arg, const_zero_node}); + const auto * if_len_node = toFunctionNode(actions_dag, "if", {less_zero_node, const_zero_node, length_arg}); + const auto * substring_func_node = toFunctionNode(actions_dag, "substringUTF8", {str_arg, if_node, if_len_node}); return convertNodeTypeIfNeeded(substrait_func, substring_func_node, actions_dag); } protected: From 06450c14eea8c3c086808967198796a6840d4cfe Mon Sep 17 00:00:00 2001 From: Chang chen Date: Tue, 4 Jun 2024 20:37:33 +0800 Subject: [PATCH 204/402] [GLUTEN-5620][CH] Simplify Decimal process for Remainder(%) operator (#5977) [CH] Simplify Decimal process for Remainder(%) operator --- .../GlutenClickHouseDecimalSuite.scala | 52 +++++++++++++++++++ .../Parser/SerializedPlanParser.cpp | 2 +- .../scalar_function_parser/arithmetic.cpp | 22 ++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala index c41ea0ccb2ea..3aa498ea35c1 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala @@ -300,6 +300,39 @@ class GlutenClickHouseDecimalSuite customCheck = customCheck, noFallBack = noFallBack) } + + test("from decimalArithmeticOperations.sql") { + // prepare + val createSql = + "create table decimals_test(id int, a decimal(38,18), b decimal(38,18)) using parquet" + val inserts = + "insert into decimals_test values(1, 100.0, 999.0)" + + ", (2, 12345.123, 12345.123)" + + ", (3, 0.1234567891011, 1234.1)" + + ", (4, 123456789123456789.0, 1.123456789123456789)" + spark.sql(createSql) + + try { + spark.sql(inserts) + + val q1 = "select id, a+b, a-b, a*b, a/b ,a%b from decimals_test order by id" + + // test operations between decimals and constants + val q2 = "select id, a*10, b/10 from decimals_test order by id" + // FIXME val q2 = "select id, a*10, b/10, a%20, b%30 from decimals_test order by id" + + Seq("true", "false").foreach { + allowPrecisionLoss => + withSQLConf((SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key, allowPrecisionLoss)) { + compareResultsAgainstVanillaSpark(q1, compareResult = true, _ => {}) + compareResultsAgainstVanillaSpark(q2, compareResult = true, _ => {}) + } + } + } finally { + spark.sql("drop table if exists decimals_test") + } + } + Seq("true", "false").foreach { allowPrecisionLoss => Range @@ -390,6 +423,25 @@ class GlutenClickHouseDecimalSuite compareResultsAgainstVanillaSpark(sql_not_null, compareResult = true, _ => {}) } + test("bigint % 6.1") { + val sql = + s""" + | select + | s_suppkey, + | s_suppkey % 6.1 + | from supplier + |""".stripMargin + spark.sql(s"use decimal_${9}_${4}") + withSQLConf(vanillaSparkConfs(): _*) { + val df2 = spark.sql(sql) + print(df2.queryExecution.executedPlan) + } + testFromRandomBase( + sql, + _ => {} + ) + } + def testFromRandomBase( sql: String, customCheck: DataFrame => Unit, diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index b0d3bbeca962..25ea86e5bec0 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -1772,7 +1772,7 @@ QueryPlanPtr SerializedPlanParser::parse(const std::string & plan) if (logger->debug()) { auto out = PlanUtil::explainPlan(*res); - LOG_ERROR(logger, "clickhouse plan:\n{}", out); + LOG_DEBUG(logger, "clickhouse plan:\n{}", out); } return res; } diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp index 2a6e435667aa..d58b22a87e6c 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arithmetic.cpp @@ -65,6 +65,13 @@ class DecimalType return bounded_to_click_house(precision, scale); } + static DecimalType evalModuloDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) + { + const Int32 scale = std::max(s1, s2); + const Int32 precision = std::min(p1 - s1, p2 - s2) + scale; + return bounded_to_click_house(precision, scale); + } + static DecimalType evalMultiplyDecimalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) { const Int32 scale = s1; @@ -221,6 +228,20 @@ class FunctionParserMultiply final : public FunctionParserBinaryArithmetic } }; +class FunctionParserModulo final : public FunctionParserBinaryArithmetic +{ +public: + explicit FunctionParserModulo(SerializedPlanParser * plan_parser_) : FunctionParserBinaryArithmetic(plan_parser_) { } + static constexpr auto name = "modulus"; + String getName() const override { return name; } + +protected: + DecimalType internalEvalType(const Int32 p1, const Int32 s1, const Int32 p2, const Int32 s2) const override + { + return DecimalType::evalModuloDecimalType(p1, s1, p2, s2); + } +}; + class FunctionParserDivide final : public FunctionParserBinaryArithmetic { public: @@ -252,5 +273,6 @@ static FunctionParserRegister register_plus; static FunctionParserRegister register_minus; static FunctionParserRegister register_mltiply; static FunctionParserRegister register_divide; +static FunctionParserRegister register_modulo; } From 9755cdb6b9756751b0f8a2c0ea519837c01e8def Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Tue, 4 Jun 2024 22:51:43 +0800 Subject: [PATCH 205/402] [VL] Fix shuffle error when null type is used (#5961) --- .../gluten/execution/TestOperator.scala | 72 +++++++++++++------ cpp/core/shuffle/Payload.cc | 2 + .../shuffle/VeloxHashBasedShuffleWriter.cc | 8 +++ 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 905d30055795..bc51ee7cb670 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1682,26 +1682,56 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } - test("Fix shuffle with round robin partitioning fail") { - def checkNullTypeRepartition(df: => DataFrame, numProject: Int): Unit = { - var expected: Array[Row] = null - withSQLConf("spark.sql.execution.sortBeforeRepartition" -> "false") { - expected = df.collect() - } - val actual = df - checkAnswer(actual, expected) - assert( - collect(actual.queryExecution.executedPlan) { case p: ProjectExec => p }.size == numProject - ) - } - - checkNullTypeRepartition( - spark.table("lineitem").selectExpr("l_orderkey", "null as x").repartition(), - 0 - ) - checkNullTypeRepartition( - spark.table("lineitem").selectExpr("null as x", "null as y").repartition(), - 1 - ) + test("Fix shuffle with null type failure") { + // single and other partitioning + Seq("1", "2").foreach { + numShufflePartitions => + withSQLConf("spark.sql.shuffle.partitions" -> numShufflePartitions) { + def checkNullTypeRepartition(df: => DataFrame, numProject: Int): Unit = { + var expected: Array[Row] = null + withSQLConf("spark.sql.execution.sortBeforeRepartition" -> "false") { + expected = df.collect() + } + val actual = df + checkAnswer(actual, expected) + assert( + collect(actual.queryExecution.executedPlan) { + case p: ProjectExec => p + }.size == numProject + ) + assert( + collect(actual.queryExecution.executedPlan) { + case shuffle: ColumnarShuffleExchangeExec => shuffle + }.size == 1 + ) + } + + // hash + checkNullTypeRepartition( + spark + .table("lineitem") + .selectExpr("l_orderkey", "null as x") + .repartition($"l_orderkey"), + 0 + ) + // range + checkNullTypeRepartition( + spark + .table("lineitem") + .selectExpr("l_orderkey", "null as x") + .repartitionByRange($"l_orderkey"), + 0 + ) + // round robin + checkNullTypeRepartition( + spark.table("lineitem").selectExpr("l_orderkey", "null as x").repartition(), + 0 + ) + checkNullTypeRepartition( + spark.table("lineitem").selectExpr("null as x", "null as y").repartition(), + 1 + ) + } + } } } diff --git a/cpp/core/shuffle/Payload.cc b/cpp/core/shuffle/Payload.cc index 626ed0cf0c18..beca3fa02d61 100644 --- a/cpp/core/shuffle/Payload.cc +++ b/cpp/core/shuffle/Payload.cc @@ -327,6 +327,8 @@ arrow::Result>> BlockPayload::deseria case arrow::ListType::type_id: { hasComplexDataType = true; } break; + case arrow::NullType::type_id: + break; default: { buffers.emplace_back(); ARROW_ASSIGN_OR_RAISE(buffers.back(), readBuffer()); diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc index daff1370332f..741ca8ab9b40 100644 --- a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc @@ -129,6 +129,14 @@ arrow::Status collectFlatVectorBufferStringView( return arrow::Status::OK(); } +template <> +arrow::Status collectFlatVectorBuffer( + facebook::velox::BaseVector* vector, + std::vector>& buffers, + arrow::MemoryPool* pool) { + return arrow::Status::OK(); +} + template <> arrow::Status collectFlatVectorBuffer( facebook::velox::BaseVector* vector, From 0f5923e9326f08c0a54139dc7b112ac30c18af5a Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Wed, 5 Jun 2024 09:31:45 +0800 Subject: [PATCH 206/402] [CORE] HashJoinLikeExecTransformer simpleStringWithNodeId adds buildSide info (#5978) --- .../tpch-approved-plan/v1-bhj-ras/spark32/10.txt | 6 +++--- .../tpch-approved-plan/v1-bhj-ras/spark32/11.txt | 4 ++-- .../tpch-approved-plan/v1-bhj-ras/spark32/12.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/13.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/14.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/15.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/16.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/18.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj-ras/spark32/19.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/20.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark32/21.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark32/22.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/3.txt | 4 ++-- .../tpch-approved-plan/v1-bhj-ras/spark32/4.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark32/5.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark32/7.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark32/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-bhj-ras/spark32/9.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark33/10.txt | 6 +++--- .../tpch-approved-plan/v1-bhj-ras/spark33/11.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj-ras/spark33/12.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/13.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/14.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/15.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/16.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/18.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj-ras/spark33/19.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/20.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark33/21.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark33/22.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/3.txt | 4 ++-- .../tpch-approved-plan/v1-bhj-ras/spark33/4.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark33/5.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark33/7.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark33/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-bhj-ras/spark33/9.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark34/10.txt | 6 +++--- .../tpch-approved-plan/v1-bhj-ras/spark34/11.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj-ras/spark34/12.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/13.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/14.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/15.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/16.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/18.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj-ras/spark34/19.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/20.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark34/21.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark34/22.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/3.txt | 4 ++-- .../tpch-approved-plan/v1-bhj-ras/spark34/4.txt | 2 +- .../tpch-approved-plan/v1-bhj-ras/spark34/5.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark34/7.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj-ras/spark34/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-bhj-ras/spark34/9.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark32/10.txt | 6 +++--- .../tpch-approved-plan/v1-bhj/spark32/11.txt | 4 ++-- .../tpch-approved-plan/v1-bhj/spark32/12.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/13.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/14.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/15.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/16.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/18.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj/spark32/19.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark32/21.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark32/22.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/3.txt | 4 ++-- .../tpch-approved-plan/v1-bhj/spark32/4.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark32/5.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark32/7.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark32/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-bhj/spark32/9.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark33/10.txt | 6 +++--- .../tpch-approved-plan/v1-bhj/spark33/11.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj/spark33/12.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/13.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/14.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/15.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/16.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/18.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj/spark33/19.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark33/21.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark33/22.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/3.txt | 4 ++-- .../tpch-approved-plan/v1-bhj/spark33/4.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark33/5.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark33/7.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark33/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-bhj/spark33/9.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark34/10.txt | 6 +++--- .../tpch-approved-plan/v1-bhj/spark34/11.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj/spark34/12.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/13.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/14.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/15.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/16.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/18.txt | 8 ++++---- .../tpch-approved-plan/v1-bhj/spark34/19.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/20.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark34/21.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark34/22.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/3.txt | 4 ++-- .../tpch-approved-plan/v1-bhj/spark34/4.txt | 2 +- .../tpch-approved-plan/v1-bhj/spark34/5.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark34/7.txt | 10 +++++----- .../tpch-approved-plan/v1-bhj/spark34/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-bhj/spark34/9.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark32/10.txt | 6 +++--- .../tpch-approved-plan/v1-ras/spark32/11.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark32/12.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/13.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/14.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/15.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/16.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/17.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark32/18.txt | 8 ++++---- .../tpch-approved-plan/v1-ras/spark32/19.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/20.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark32/21.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark32/22.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/3.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark32/4.txt | 2 +- .../tpch-approved-plan/v1-ras/spark32/5.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark32/7.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark32/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark33/10.txt | 6 +++--- .../tpch-approved-plan/v1-ras/spark33/11.txt | 8 ++++---- .../tpch-approved-plan/v1-ras/spark33/12.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/13.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/14.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/15.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/16.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/17.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark33/18.txt | 8 ++++---- .../tpch-approved-plan/v1-ras/spark33/19.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/20.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark33/21.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark33/22.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/3.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark33/4.txt | 2 +- .../tpch-approved-plan/v1-ras/spark33/5.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark33/7.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark33/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark34/10.txt | 6 +++--- .../tpch-approved-plan/v1-ras/spark34/11.txt | 8 ++++---- .../tpch-approved-plan/v1-ras/spark34/12.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/13.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/14.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/15.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/16.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/17.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark34/18.txt | 8 ++++---- .../tpch-approved-plan/v1-ras/spark34/19.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/20.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark34/21.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark34/22.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/3.txt | 4 ++-- .../tpch-approved-plan/v1-ras/spark34/4.txt | 2 +- .../tpch-approved-plan/v1-ras/spark34/5.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark34/7.txt | 10 +++++----- .../tpch-approved-plan/v1-ras/spark34/8.txt | 14 +++++++------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark32/10.txt | 6 +++--- .../resources/tpch-approved-plan/v1/spark32/11.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark32/12.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/13.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/14.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/15.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/16.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/17.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark32/18.txt | 8 ++++---- .../resources/tpch-approved-plan/v1/spark32/19.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/20.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark32/21.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark32/22.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/3.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark32/4.txt | 2 +- .../resources/tpch-approved-plan/v1/spark32/5.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark32/7.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark32/8.txt | 14 +++++++------- .../resources/tpch-approved-plan/v1/spark32/9.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark33/10.txt | 6 +++--- .../resources/tpch-approved-plan/v1/spark33/11.txt | 8 ++++---- .../resources/tpch-approved-plan/v1/spark33/12.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/13.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/14.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/15.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/16.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/17.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark33/18.txt | 8 ++++---- .../resources/tpch-approved-plan/v1/spark33/19.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/20.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark33/21.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark33/22.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/3.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark33/4.txt | 2 +- .../resources/tpch-approved-plan/v1/spark33/5.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark33/7.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark33/8.txt | 14 +++++++------- .../resources/tpch-approved-plan/v1/spark33/9.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark34/10.txt | 6 +++--- .../resources/tpch-approved-plan/v1/spark34/11.txt | 8 ++++---- .../resources/tpch-approved-plan/v1/spark34/12.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/13.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/14.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/15.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/16.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/17.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark34/18.txt | 8 ++++---- .../resources/tpch-approved-plan/v1/spark34/19.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/20.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark34/21.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark34/22.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/3.txt | 4 ++-- .../resources/tpch-approved-plan/v1/spark34/4.txt | 2 +- .../resources/tpch-approved-plan/v1/spark34/5.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark34/7.txt | 10 +++++----- .../resources/tpch-approved-plan/v1/spark34/8.txt | 14 +++++++------- .../resources/tpch-approved-plan/v1/spark34/9.txt | 10 +++++----- .../gluten/execution/JoinExecTransformer.scala | 7 ++++++- .../execution/SortMergeJoinExecTransformer.scala | 6 ------ 224 files changed, 646 insertions(+), 647 deletions(-) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index c0153da0cda9..512aa92207cd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -12,11 +12,11 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (33) +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) : : :- ^ NoopFilter (2) : : : +- ^ Scan parquet (1) : : +- ^ InputIteratorTransformer (10) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index 219d4b7c14de..a330846ccf90 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (58) +- ^ ProjectExecTransformer (23) +- ^ FlushableHashAggregateExecTransformer (22) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) : :- ^ NoopFilter (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index 985cbf4c3c59..dea6e04c33ff 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (14) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 4a3a239381ca..82786dc4457b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -23,7 +23,7 @@ AdaptiveSparkPlan (52) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) :- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) +- ^ InputAdapter (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index ee1fe0a3316c..7de47d57f6fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (34) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index 076e82bd53e3..c16f304be090 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (44) +- ^ ShuffleQueryStage (24) +- ColumnarExchange (23) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index 603ea771b167..08eb4f850f22 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (56) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) :- ^ NoopFilter (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index c294e8e6dac1..8dd094081ad3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -11,16 +11,16 @@ AdaptiveSparkPlan (86) +- ^ ProjectExecTransformer (44) +- ^ FlushableHashAggregateExecTransformer (43) +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : :- ^ InputIteratorTransformer (7) : : +- ^ InputAdapter (6) : : +- ^ BroadcastQueryStage (5) : : +- ColumnarBroadcastExchange (4) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ NoopFilter (9) : : +- ^ Scan parquet (8) : +- ^ InputIteratorTransformer (25) @@ -41,7 +41,7 @@ AdaptiveSparkPlan (86) +- ^ InputAdapter (39) +- ^ BroadcastQueryStage (38) +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) :- ^ NoopFilter (30) : +- ^ Scan parquet (29) +- ^ InputIteratorTransformer (34) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 267ae6add0ac..289def220b08 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (33) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index 1b4522028bb4..71889beea972 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -8,9 +8,9 @@ AdaptiveSparkPlan (107) +- ^ ShuffleQueryStage (65) +- ColumnarExchange (64) +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner (61) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) : :- ^ InputIteratorTransformer (9) : : +- ^ InputAdapter (8) : : +- ^ AQEShuffleRead (7) @@ -24,12 +24,12 @@ AdaptiveSparkPlan (107) : +- ^ BroadcastQueryStage (48) : +- ColumnarBroadcastExchange (47) : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) : :- ^ InputIteratorTransformer (25) : : +- ^ InputAdapter (24) : : +- ^ BroadcastQueryStage (23) : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) : : :- ^ NoopFilter (11) : : : +- ^ Scan parquet (10) : : +- ^ InputIteratorTransformer (19) @@ -48,7 +48,7 @@ AdaptiveSparkPlan (107) : +- ColumnarExchange (37) : +- ^ ProjectExecTransformer (35) : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) : :- ^ ProjectExecTransformer (28) : : +- ^ NoopFilter (27) : : +- ^ Scan parquet (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index 9c09032689eb..bd977b45da55 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -11,19 +11,19 @@ AdaptiveSparkPlan (92) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : : :- ^ InputIteratorTransformer (7) : : : +- ^ InputAdapter (6) : : : +- ^ BroadcastQueryStage (5) : : : +- ColumnarBroadcastExchange (4) : : : +- ^ NoopFilter (2) : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) : : : :- ^ ProjectExecTransformer (10) : : : : +- ^ NoopFilter (9) : : : : +- ^ Scan parquet (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index 3dc92be2d96e..4b7d882ca23e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (38) +- ^ ProjectExecTransformer (12) +- ^ FlushableHashAggregateExecTransformer (11) +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) :- ^ NoopFilter (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index f962dbdd765f..e5f266cc71f3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -12,9 +12,9 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (24) +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ BroadcastQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index e50973eb5abf..2bb4d8907de7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (44) +- ^ ProjectExecTransformer (15) +- ^ FlushableHashAggregateExecTransformer (14) +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index 274c176ff8c8..6ce90735fbeb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (100) +- ^ ProjectExecTransformer (51) +- ^ FlushableHashAggregateExecTransformer (50) +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index 3d5b6496fe27..ae4b171974b1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (93) +- ^ ProjectExecTransformer (46) +- ^ FlushableHashAggregateExecTransformer (45) +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 9489452b5272..ce470033eb1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -16,19 +16,19 @@ AdaptiveSparkPlan (129) +- ^ ProjectExecTransformer (69) +- ^ FlushableHashAggregateExecTransformer (68) +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ BroadcastQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index 29e6d2f72f57..3744b8895539 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (98) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ BroadcastQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 276588749734..0c1832920c86 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -12,11 +12,11 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (33) +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) : : :- ^ NoopFilter (2) : : : +- ^ Scan parquet (1) : : +- ^ InputIteratorTransformer (10) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 54f535dfbfb6..b66d0bb6a930 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (58) +- ^ ProjectExecTransformer (23) +- ^ FlushableHashAggregateExecTransformer (22) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) : :- ^ NoopFilter (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (9) @@ -327,9 +327,9 @@ AdaptiveSparkPlan (99) +- ColumnarExchange (75) +- ^ FlushableHashAggregateExecTransformer (73) +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner (71) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) : :- ^ NoopFilter (60) : : +- ^ Scan parquet (59) : +- ^ InputIteratorTransformer (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index 1f1fcd3fb577..073b9de3885e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (14) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index e67819b82860..baa569ccebe9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -23,7 +23,7 @@ AdaptiveSparkPlan (52) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) :- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) +- ^ InputAdapter (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index eb04b6c18271..699ef76d14e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (34) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 5c6304211102..fdb22283555b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (41) +- ShuffleQueryStage (24), Statistics(X) +- ColumnarExchange (23) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index 9a4005e3da85..05b0b2f9723a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (56) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) :- ^ NoopFilter (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 5a941687c525..c21e377b0153 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -11,16 +11,16 @@ AdaptiveSparkPlan (86) +- ^ ProjectExecTransformer (44) +- ^ FlushableHashAggregateExecTransformer (43) +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : :- ^ InputIteratorTransformer (7) : : +- ^ InputAdapter (6) : : +- ^ BroadcastQueryStage (5), Statistics(X) : : +- ColumnarBroadcastExchange (4) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ NoopFilter (9) : : +- ^ Scan parquet (8) : +- ^ InputIteratorTransformer (25) @@ -41,7 +41,7 @@ AdaptiveSparkPlan (86) +- ^ InputAdapter (39) +- ^ BroadcastQueryStage (38), Statistics(X) +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) :- ^ NoopFilter (30) : +- ^ Scan parquet (29) +- ^ InputIteratorTransformer (34) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 244f1c6ffd89..bce0e46e72dc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (33) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index fd6deaabc6f5..d4aec5f07beb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (104) +- ShuffleQueryStage (65), Statistics(X) +- ColumnarExchange (64) +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner (61) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) : :- ^ InputIteratorTransformer (9) : : +- ^ InputAdapter (8) : : +- ^ AQEShuffleRead (7) @@ -22,12 +22,12 @@ AdaptiveSparkPlan (104) : +- ^ BroadcastQueryStage (48), Statistics(X) : +- ColumnarBroadcastExchange (47) : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) : :- ^ InputIteratorTransformer (25) : : +- ^ InputAdapter (24) : : +- ^ BroadcastQueryStage (23), Statistics(X) : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) : : :- ^ NoopFilter (11) : : : +- ^ Scan parquet (10) : : +- ^ InputIteratorTransformer (19) @@ -46,7 +46,7 @@ AdaptiveSparkPlan (104) : +- ColumnarExchange (37) : +- ^ ProjectExecTransformer (35) : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) : :- ^ ProjectExecTransformer (28) : : +- ^ NoopFilter (27) : : +- ^ Scan parquet (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index ac047fdbcabf..361860c7486b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (91) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : : :- ^ InputIteratorTransformer (7) : : : +- ^ InputAdapter (6) : : : +- ^ BroadcastQueryStage (5), Statistics(X) : : : +- ColumnarBroadcastExchange (4) : : : +- ^ NoopFilter (2) : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) : : : :- ^ ProjectExecTransformer (10) : : : : +- ^ NoopFilter (9) : : : : +- ^ Scan parquet (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index 3306ea6b2fd9..c0a11106bbf9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (38) +- ^ ProjectExecTransformer (12) +- ^ FlushableHashAggregateExecTransformer (11) +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) :- ^ NoopFilter (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index 0bcfb2c8228d..153e81420430 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -12,9 +12,9 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (24) +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index b547b4051bcf..ccb61f1c6cd3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (44) +- ^ ProjectExecTransformer (15) +- ^ FlushableHashAggregateExecTransformer (14) +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index a87ef7c618c0..b12ece606bc2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (100) +- ^ ProjectExecTransformer (51) +- ^ FlushableHashAggregateExecTransformer (50) +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 37e9e5214256..3fec6f0c8b2c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (93) +- ^ ProjectExecTransformer (46) +- ^ FlushableHashAggregateExecTransformer (45) +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 9b490b9ba436..da6e7a768f23 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -16,19 +16,19 @@ AdaptiveSparkPlan (129) +- ^ ProjectExecTransformer (69) +- ^ FlushableHashAggregateExecTransformer (68) +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 8523cbeff2ab..6e93f3a79f22 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (98) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index 276a885015d0..b917f359d1a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -12,11 +12,11 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (33) +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) : : :- ^ NoopFilter (2) : : : +- ^ Scan parquet (1) : : +- ^ InputIteratorTransformer (10) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index f8d59a67a22b..50c599a3b051 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (58) +- ^ ProjectExecTransformer (23) +- ^ FlushableHashAggregateExecTransformer (22) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) : :- ^ NoopFilter (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (9) @@ -331,9 +331,9 @@ AdaptiveSparkPlan (99) +- ColumnarExchange (75) +- ^ FlushableHashAggregateExecTransformer (73) +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner (71) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) : :- ^ NoopFilter (60) : : +- ^ Scan parquet (59) : +- ^ InputIteratorTransformer (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index 9e142ea961f5..c57f98c1ff85 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (14) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index 9841ce1395e1..b29e27ae092b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -23,7 +23,7 @@ AdaptiveSparkPlan (52) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) :- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) +- ^ InputAdapter (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index fe13deb40500..4b3e5634106f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (34) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 4b761198f3e3..27eec9ac6340 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (41) +- ShuffleQueryStage (24), Statistics(X) +- ColumnarExchange (23) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index f178803b43e9..fdec20383624 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (56) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) :- ^ NoopFilter (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index 2484955032d0..66819677c834 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -11,16 +11,16 @@ AdaptiveSparkPlan (86) +- ^ ProjectExecTransformer (44) +- ^ FlushableHashAggregateExecTransformer (43) +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : :- ^ InputIteratorTransformer (7) : : +- ^ InputAdapter (6) : : +- ^ BroadcastQueryStage (5), Statistics(X) : : +- ColumnarBroadcastExchange (4) : : +- ^ NoopFilter (2) : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ NoopFilter (9) : : +- ^ Scan parquet (8) : +- ^ InputIteratorTransformer (25) @@ -41,7 +41,7 @@ AdaptiveSparkPlan (86) +- ^ InputAdapter (39) +- ^ BroadcastQueryStage (38), Statistics(X) +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) :- ^ NoopFilter (30) : +- ^ Scan parquet (29) +- ^ InputIteratorTransformer (34) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index 1a2a13c21db8..9b8041d6e824 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (33) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index ed40d3bbe23d..e1724af1c30c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (96) +- ShuffleQueryStage (58), Statistics(X) +- ColumnarExchange (57) +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner (54) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (44) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ NoopFilter (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (43) @@ -16,12 +16,12 @@ AdaptiveSparkPlan (96) : +- ^ BroadcastQueryStage (41), Statistics(X) : +- ColumnarBroadcastExchange (40) : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) : :- ^ InputIteratorTransformer (18) : : +- ^ InputAdapter (17) : : +- ^ BroadcastQueryStage (16), Statistics(X) : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (13) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) : : :- ^ NoopFilter (4) : : : +- ^ Scan parquet (3) : : +- ^ InputIteratorTransformer (12) @@ -40,7 +40,7 @@ AdaptiveSparkPlan (96) : +- ColumnarExchange (30) : +- ^ ProjectExecTransformer (28) : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ ProjectExecTransformer (21) : : +- ^ NoopFilter (20) : : +- ^ Scan parquet (19) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index d0b14137ca5a..c418b8c4fba4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (91) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : : :- ^ InputIteratorTransformer (7) : : : +- ^ InputAdapter (6) : : : +- ^ BroadcastQueryStage (5), Statistics(X) : : : +- ColumnarBroadcastExchange (4) : : : +- ^ NoopFilter (2) : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) : : : :- ^ ProjectExecTransformer (10) : : : : +- ^ NoopFilter (9) : : : : +- ^ Scan parquet (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 698879c473b6..953ec6a34a55 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (38) +- ^ ProjectExecTransformer (12) +- ^ FlushableHashAggregateExecTransformer (11) +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) :- ^ NoopFilter (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index 98a7cd0e0145..af59560de7d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -12,9 +12,9 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (24) +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index b5fefc6bef3c..1e7d5f6793cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (44) +- ^ ProjectExecTransformer (15) +- ^ FlushableHashAggregateExecTransformer (14) +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) :- ^ ProjectExecTransformer (3) : +- ^ NoopFilter (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index be0403f587f5..170e4d012be3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (100) +- ^ ProjectExecTransformer (51) +- ^ FlushableHashAggregateExecTransformer (50) +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 2ac2968387fd..9b6036312f19 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (93) +- ^ ProjectExecTransformer (46) +- ^ FlushableHashAggregateExecTransformer (45) +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index 65906bb96691..48924df3085b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -16,19 +16,19 @@ AdaptiveSparkPlan (129) +- ^ ProjectExecTransformer (69) +- ^ FlushableHashAggregateExecTransformer (68) +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index e7abd01744e5..7e7b4390eefe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (98) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index 935650e26059..c6d18d83e37a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -12,11 +12,11 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (33) +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) : : :- ^ FilterExecTransformer (2) : : : +- ^ Scan parquet (1) : : +- ^ InputIteratorTransformer (10) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt index 1cc9e9ac278e..c147c286d9cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (58) +- ^ ProjectExecTransformer (23) +- ^ FlushableHashAggregateExecTransformer (22) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) : :- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt index 1d477cbd774e..cdd0b5527b0c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (14) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt index 492cfafb1ec1..a0733090396d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt @@ -23,7 +23,7 @@ AdaptiveSparkPlan (52) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) :- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) +- ^ InputAdapter (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt index b564240ff552..567e2ba8fa8f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (34) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt index ff632e930f72..1d63d95f9c29 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (44) +- ^ ShuffleQueryStage (24) +- ColumnarExchange (23) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt index 06fcb53dfebe..f9c811ab7bd1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (56) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) :- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index 32359df26265..53d2e12db19b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -11,16 +11,16 @@ AdaptiveSparkPlan (86) +- ^ ProjectExecTransformer (44) +- ^ FlushableHashAggregateExecTransformer (43) +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : :- ^ InputIteratorTransformer (7) : : +- ^ InputAdapter (6) : : +- ^ BroadcastQueryStage (5) : : +- ColumnarBroadcastExchange (4) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ FilterExecTransformer (9) : : +- ^ Scan parquet (8) : +- ^ InputIteratorTransformer (25) @@ -41,7 +41,7 @@ AdaptiveSparkPlan (86) +- ^ InputAdapter (39) +- ^ BroadcastQueryStage (38) +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) :- ^ FilterExecTransformer (30) : +- ^ Scan parquet (29) +- ^ InputIteratorTransformer (34) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt index 232c27fb8a9e..15cc941c36de 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (33) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index fe2ae1df7cc7..8af0e8b1d511 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -8,9 +8,9 @@ AdaptiveSparkPlan (107) +- ^ ShuffleQueryStage (65) +- ColumnarExchange (64) +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner (61) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) : :- ^ InputIteratorTransformer (9) : : +- ^ InputAdapter (8) : : +- ^ AQEShuffleRead (7) @@ -24,12 +24,12 @@ AdaptiveSparkPlan (107) : +- ^ BroadcastQueryStage (48) : +- ColumnarBroadcastExchange (47) : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) : :- ^ InputIteratorTransformer (25) : : +- ^ InputAdapter (24) : : +- ^ BroadcastQueryStage (23) : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) : : :- ^ FilterExecTransformer (11) : : : +- ^ Scan parquet (10) : : +- ^ InputIteratorTransformer (19) @@ -48,7 +48,7 @@ AdaptiveSparkPlan (107) : +- ColumnarExchange (37) : +- ^ ProjectExecTransformer (35) : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) : :- ^ ProjectExecTransformer (28) : : +- ^ FilterExecTransformer (27) : : +- ^ Scan parquet (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index c3bd62e04735..bf33da8261c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -11,19 +11,19 @@ AdaptiveSparkPlan (92) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : : :- ^ InputIteratorTransformer (7) : : : +- ^ InputAdapter (6) : : : +- ^ BroadcastQueryStage (5) : : : +- ColumnarBroadcastExchange (4) : : : +- ^ FilterExecTransformer (2) : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) : : : :- ^ ProjectExecTransformer (10) : : : : +- ^ FilterExecTransformer (9) : : : : +- ^ Scan parquet (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt index dff492685686..3723f8f01625 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (38) +- ^ ProjectExecTransformer (12) +- ^ FlushableHashAggregateExecTransformer (11) +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) :- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index 950a881d1ecf..373882a24e57 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -12,9 +12,9 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (24) +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ BroadcastQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt index 3969248bc2ec..adf7259779a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (44) +- ^ ProjectExecTransformer (15) +- ^ FlushableHashAggregateExecTransformer (14) +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt index 678a52004a3f..cfed4af71090 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (100) +- ^ ProjectExecTransformer (51) +- ^ FlushableHashAggregateExecTransformer (50) +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt index 19b6cc4b8aee..910190a65e0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (93) +- ^ ProjectExecTransformer (46) +- ^ FlushableHashAggregateExecTransformer (45) +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt index 3aa6e39645cc..f16be6bdb2e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt @@ -16,19 +16,19 @@ AdaptiveSparkPlan (129) +- ^ ProjectExecTransformer (69) +- ^ FlushableHashAggregateExecTransformer (68) +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ BroadcastQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt index b00947a3e3e0..22233dbc5ae2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (98) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ BroadcastQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index 2977896f3da8..ebb947ebb002 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -12,11 +12,11 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (33) +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) : : :- ^ FilterExecTransformer (2) : : : +- ^ Scan parquet (1) : : +- ^ InputIteratorTransformer (10) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt index 2a860ed5c063..6ab05c3c2261 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (58) +- ^ ProjectExecTransformer (23) +- ^ FlushableHashAggregateExecTransformer (22) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) : :- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (9) @@ -327,9 +327,9 @@ AdaptiveSparkPlan (99) +- ColumnarExchange (75) +- ^ FlushableHashAggregateExecTransformer (73) +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner (71) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) : :- ^ FilterExecTransformer (60) : : +- ^ Scan parquet (59) : +- ^ InputIteratorTransformer (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt index 99576f6e063a..98272b12cebc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (14) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt index 8ebeb7d32d52..c1919f2e620d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt @@ -23,7 +23,7 @@ AdaptiveSparkPlan (52) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) :- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) +- ^ InputAdapter (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt index bfba707a5792..17283b824590 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (34) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt index 06b833361548..ce87f94218ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (41) +- ShuffleQueryStage (24), Statistics(X) +- ColumnarExchange (23) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt index af623a3a2f68..6a460d28f171 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (56) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) :- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index cc13e5f3b886..884bba49cbf3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -11,16 +11,16 @@ AdaptiveSparkPlan (86) +- ^ ProjectExecTransformer (44) +- ^ FlushableHashAggregateExecTransformer (43) +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : :- ^ InputIteratorTransformer (7) : : +- ^ InputAdapter (6) : : +- ^ BroadcastQueryStage (5), Statistics(X) : : +- ColumnarBroadcastExchange (4) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ FilterExecTransformer (9) : : +- ^ Scan parquet (8) : +- ^ InputIteratorTransformer (25) @@ -41,7 +41,7 @@ AdaptiveSparkPlan (86) +- ^ InputAdapter (39) +- ^ BroadcastQueryStage (38), Statistics(X) +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) :- ^ FilterExecTransformer (30) : +- ^ Scan parquet (29) +- ^ InputIteratorTransformer (34) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt index 3350a6c3009f..b78310babb5b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (33) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index a95115d00b26..f71a534d99d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (104) +- ShuffleQueryStage (65), Statistics(X) +- ColumnarExchange (64) +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner (61) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) : :- ^ InputIteratorTransformer (9) : : +- ^ InputAdapter (8) : : +- ^ AQEShuffleRead (7) @@ -22,12 +22,12 @@ AdaptiveSparkPlan (104) : +- ^ BroadcastQueryStage (48), Statistics(X) : +- ColumnarBroadcastExchange (47) : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner (44) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) : :- ^ InputIteratorTransformer (25) : : +- ^ InputAdapter (24) : : +- ^ BroadcastQueryStage (23), Statistics(X) : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) : : :- ^ FilterExecTransformer (11) : : : +- ^ Scan parquet (10) : : +- ^ InputIteratorTransformer (19) @@ -46,7 +46,7 @@ AdaptiveSparkPlan (104) : +- ColumnarExchange (37) : +- ^ ProjectExecTransformer (35) : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) : :- ^ ProjectExecTransformer (28) : : +- ^ FilterExecTransformer (27) : : +- ^ Scan parquet (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt index 2e1ddb4e0851..46792bf54410 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (91) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : : :- ^ InputIteratorTransformer (7) : : : +- ^ InputAdapter (6) : : : +- ^ BroadcastQueryStage (5), Statistics(X) : : : +- ColumnarBroadcastExchange (4) : : : +- ^ FilterExecTransformer (2) : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) : : : :- ^ ProjectExecTransformer (10) : : : : +- ^ FilterExecTransformer (9) : : : : +- ^ Scan parquet (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt index 774b9ca85165..38d759bb1020 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (38) +- ^ ProjectExecTransformer (12) +- ^ FlushableHashAggregateExecTransformer (11) +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) :- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 6a5c8395fa5d..0b8395a482ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -12,9 +12,9 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (24) +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt index 0bbf78a26ac8..a9b7bac67fc4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (44) +- ^ ProjectExecTransformer (15) +- ^ FlushableHashAggregateExecTransformer (14) +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt index cc6561aefb3b..fb55e067f7e9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (100) +- ^ ProjectExecTransformer (51) +- ^ FlushableHashAggregateExecTransformer (50) +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt index bab148b6bb17..4c62f3ce9132 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (93) +- ^ ProjectExecTransformer (46) +- ^ FlushableHashAggregateExecTransformer (45) +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt index 757b1fcca6b4..5d4ef3143b4f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt @@ -16,19 +16,19 @@ AdaptiveSparkPlan (129) +- ^ ProjectExecTransformer (69) +- ^ FlushableHashAggregateExecTransformer (68) +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt index 4b0cb7140dcd..227e77ecc700 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (98) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index a0a2ead10a37..d6b77aaf9db7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -12,11 +12,11 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (33) +- ^ FlushableHashAggregateExecTransformer (32) +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) : : :- ^ FilterExecTransformer (2) : : : +- ^ Scan parquet (1) : : +- ^ InputIteratorTransformer (10) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt index c81d951f6e31..8ee504fafda7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (58) +- ^ ProjectExecTransformer (23) +- ^ FlushableHashAggregateExecTransformer (22) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) : :- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (9) @@ -331,9 +331,9 @@ AdaptiveSparkPlan (99) +- ColumnarExchange (75) +- ^ FlushableHashAggregateExecTransformer (73) +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner (71) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner (65) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) : :- ^ FilterExecTransformer (60) : : +- ^ Scan parquet (59) : +- ^ InputIteratorTransformer (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt index 36ebd2bf5e76..55be10b8eb23 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (14) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt index aa873582fab2..efeb21b15e68 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt @@ -23,7 +23,7 @@ AdaptiveSparkPlan (52) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) :- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) +- ^ InputAdapter (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt index 3b5202f36c1e..f4f21eafc08a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt @@ -10,7 +10,7 @@ AdaptiveSparkPlan (34) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt index 0fd34791c150..baf2075cf671 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (41) +- ShuffleQueryStage (24), Statistics(X) +- ColumnarExchange (23) +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt index ed932e494f66..13f5405e5091 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (56) +- ^ ProjectExecTransformer (13) +- ^ FlushableHashAggregateExecTransformer (12) +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) :- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (9) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 0969c7267af5..09ceccba8b0a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -11,16 +11,16 @@ AdaptiveSparkPlan (86) +- ^ ProjectExecTransformer (44) +- ^ FlushableHashAggregateExecTransformer (43) +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : :- ^ InputIteratorTransformer (7) : : +- ^ InputAdapter (6) : : +- ^ BroadcastQueryStage (5), Statistics(X) : : +- ColumnarBroadcastExchange (4) : : +- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ FilterExecTransformer (9) : : +- ^ Scan parquet (8) : +- ^ InputIteratorTransformer (25) @@ -41,7 +41,7 @@ AdaptiveSparkPlan (86) +- ^ InputAdapter (39) +- ^ BroadcastQueryStage (38), Statistics(X) +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) :- ^ FilterExecTransformer (30) : +- ^ Scan parquet (29) +- ^ InputIteratorTransformer (34) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt index 87f00043a52a..ded467ae7006 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt @@ -9,7 +9,7 @@ AdaptiveSparkPlan (33) +- ColumnarExchange (15) +- ^ FlushableHashAggregateExecTransformer (13) +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt index 5db32ebb4227..e1ce25665b3e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (96) +- ShuffleQueryStage (58), Statistics(X) +- ColumnarExchange (57) +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner (54) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (44) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (43) @@ -16,12 +16,12 @@ AdaptiveSparkPlan (96) : +- ^ BroadcastQueryStage (41), Statistics(X) : +- ColumnarBroadcastExchange (40) : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) : :- ^ InputIteratorTransformer (18) : : +- ^ InputAdapter (17) : : +- ^ BroadcastQueryStage (16), Statistics(X) : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (13) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) : : :- ^ FilterExecTransformer (4) : : : +- ^ Scan parquet (3) : : +- ^ InputIteratorTransformer (12) @@ -40,7 +40,7 @@ AdaptiveSparkPlan (96) : +- ColumnarExchange (30) : +- ^ ProjectExecTransformer (28) : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) : :- ^ ProjectExecTransformer (21) : : +- ^ FilterExecTransformer (20) : : +- ^ Scan parquet (19) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt index 6e415d4a7c38..31a2b07b31f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (91) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) : : :- ^ InputIteratorTransformer (7) : : : +- ^ InputAdapter (6) : : : +- ^ BroadcastQueryStage (5), Statistics(X) : : : +- ColumnarBroadcastExchange (4) : : : +- ^ FilterExecTransformer (2) : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) : : : :- ^ ProjectExecTransformer (10) : : : : +- ^ FilterExecTransformer (9) : : : : +- ^ Scan parquet (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt index b0b5964d0a94..bfc52b25c52a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (38) +- ^ ProjectExecTransformer (12) +- ^ FlushableHashAggregateExecTransformer (11) +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) :- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (8) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index 6ce73eb0fa38..fdde00c0111b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -12,9 +12,9 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (24) +- ^ FlushableHashAggregateExecTransformer (23) +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt index 2401dc071221..a423034982a9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (44) +- ^ ProjectExecTransformer (15) +- ^ FlushableHashAggregateExecTransformer (14) +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) :- ^ ProjectExecTransformer (3) : +- ^ FilterExecTransformer (2) : +- ^ Scan parquet (1) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt index e811b6370366..6ce78c899186 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (100) +- ^ ProjectExecTransformer (51) +- ^ FlushableHashAggregateExecTransformer (50) +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt index 129da90d3a2b..3a43310c0813 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (93) +- ^ ProjectExecTransformer (46) +- ^ FlushableHashAggregateExecTransformer (45) +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) : : : : :- ^ InputIteratorTransformer (7) : : : : : +- ^ InputAdapter (6) : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt index 4ac782958199..97a1de2e4861 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt @@ -16,19 +16,19 @@ AdaptiveSparkPlan (129) +- ^ ProjectExecTransformer (69) +- ^ FlushableHashAggregateExecTransformer (68) +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt index 1a41ac753c5c..8003b4e2a7c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt @@ -15,15 +15,15 @@ AdaptiveSparkPlan (98) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 9af231ec1bb8..a545d26e2b12 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -12,19 +12,19 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) :- ^ InputIteratorTransformer (38) : +- ^ InputAdapter (37) : +- ^ ShuffleQueryStage (36) : +- ColumnarExchange (35) : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : :- ^ InputIteratorTransformer (23) : : +- ^ InputAdapter (22) : : +- ^ ShuffleQueryStage (21) : : +- ColumnarExchange (20) : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index f028b8daa9b3..717acdacccd4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -16,13 +16,13 @@ AdaptiveSparkPlan (72) +- ^ ProjectExecTransformer (35) +- ^ FlushableHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index 914b1813df44..ff48c00b1d85 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (49) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index 408b8a0f7f97..48e48c97477f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -18,7 +18,7 @@ AdaptiveSparkPlan (52) +- ^ RegularHashAggregateExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index c1e2a1e52130..c4893bc66912 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (35) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 1da11e4fffea..03aaba455979 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (45) +- ^ ShuffleQueryStage (25) +- ColumnarExchange (24) +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 4a8b56e41f36..be28ff7e84eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (64) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 523c081b1d19..e2ff1fcc0eb5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (57) +- ^ RegularHashAggregateExecTransformer (34) +- ^ RegularHashAggregateExecTransformer (33) +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 6045020cb49a..3b6719e7e55d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (97) +- ^ RegularHashAggregateExecTransformer (61) +- ^ RegularHashAggregateExecTransformer (60) +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) : +- ^ InputAdapter (40) : +- ^ ShuffleQueryStage (39) : +- ColumnarExchange (38) : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) @@ -25,7 +25,7 @@ AdaptiveSparkPlan (97) : +- ^ ShuffleQueryStage (32) : +- ColumnarExchange (31) : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) : :- ^ InputIteratorTransformer (16) : : +- ^ InputAdapter (15) : : +- ^ ShuffleQueryStage (14) @@ -43,7 +43,7 @@ AdaptiveSparkPlan (97) : +- ^ ProjectExecTransformer (19) : +- ^ FlushableHashAggregateExecTransformer (18) : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) : +- ^ InputAdapter (48) : +- ^ ShuffleQueryStage (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index 7084e61f61d5..290d398f4edd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -5,7 +5,7 @@ AdaptiveSparkPlan (34) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 3f6d17512477..bdf5d16a8fd3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -8,13 +8,13 @@ AdaptiveSparkPlan (126) +- ^ ShuffleQueryStage (81) +- ColumnarExchange (80) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) @@ -27,13 +27,13 @@ AdaptiveSparkPlan (126) : +- ^ ShuffleQueryStage (59) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) : :- ^ InputIteratorTransformer (31) : : +- ^ InputAdapter (30) : : +- ^ ShuffleQueryStage (29) : : +- ColumnarExchange (28) : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) : : :- ^ InputIteratorTransformer (16) : : : +- ^ InputAdapter (15) : : : +- ^ ShuffleQueryStage (14) @@ -57,7 +57,7 @@ AdaptiveSparkPlan (126) : +- ^ ProjectExecTransformer (47) : +- ^ RegularHashAggregateExecTransformer (46) : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ InputIteratorTransformer (39) : : +- ^ InputAdapter (38) : : +- ^ ShuffleQueryStage (37) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index b7067f8b5e3a..b960a7682beb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -11,19 +11,19 @@ AdaptiveSparkPlan (119) +- ^ ProjectExecTransformer (73) +- ^ FlushableHashAggregateExecTransformer (72) +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) :- ^ InputIteratorTransformer (61) : +- ^ InputAdapter (60) : +- ^ ShuffleQueryStage (59) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) : :- ^ InputIteratorTransformer (46) : : +- ^ InputAdapter (45) : : +- ^ ShuffleQueryStage (44) : : +- ColumnarExchange (43) : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6) @@ -36,8 +36,8 @@ AdaptiveSparkPlan (119) : : +- ^ ShuffleQueryStage (37) : : +- ColumnarExchange (36) : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) : : : :- ^ InputIteratorTransformer (16) : : : : +- ^ InputAdapter (15) : : : : +- ^ ShuffleQueryStage (14) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index dc38dbb7675a..67b2d945c059 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (46) +- ^ ProjectExecTransformer (19) +- ^ FlushableHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index d6c6f13e9063..6061266d1ab3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -7,13 +7,13 @@ AdaptiveSparkPlan (59) +- ^ RegularHashAggregateExecTransformer (35) +- ^ RegularHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 4c33fb73f757..be03c6970a0a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index 8bc2587dbaee..a7093229bb11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (134) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 5ac66a7e2708..132fd068b4d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (128) +- ^ ProjectExecTransformer (76) +- ^ FlushableHashAggregateExecTransformer (75) +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 7e7064f9fbae..043826c912d8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -16,43 +16,43 @@ AdaptiveSparkPlan (177) +- ^ ProjectExecTransformer (110) +- ^ FlushableHashAggregateExecTransformer (109) +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) :- ^ InputIteratorTransformer (98) : +- ^ InputAdapter (97) : +- ^ ShuffleQueryStage (96) : +- ColumnarExchange (95) : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) : :- ^ InputIteratorTransformer (83) : : +- ^ InputAdapter (82) : : +- ^ ShuffleQueryStage (81) : : +- ColumnarExchange (80) : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) : : :- ^ InputIteratorTransformer (68) : : : +- ^ InputAdapter (67) : : : +- ^ ShuffleQueryStage (66) : : : +- ColumnarExchange (65) : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : : : :- ^ InputIteratorTransformer (53) : : : : +- ^ InputAdapter (52) : : : : +- ^ ShuffleQueryStage (51) : : : : +- ColumnarExchange (50) : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : : : :- ^ InputIteratorTransformer (38) : : : : : +- ^ InputAdapter (37) : : : : : +- ^ ShuffleQueryStage (36) : : : : : +- ColumnarExchange (35) : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : : : :- ^ InputIteratorTransformer (23) : : : : : : +- ^ InputAdapter (22) : : : : : : +- ^ ShuffleQueryStage (21) : : : : : : +- ColumnarExchange (20) : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 5ba4a9f7ce0e..bc8752ca1f2d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (133) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 66b2ccdc77b3..5050dab789f0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -12,19 +12,19 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) :- ^ InputIteratorTransformer (38) : +- ^ InputAdapter (37) : +- ^ ShuffleQueryStage (36), Statistics(X) : +- ColumnarExchange (35) : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : :- ^ InputIteratorTransformer (23) : : +- ^ InputAdapter (22) : : +- ^ ShuffleQueryStage (21), Statistics(X) : : +- ColumnarExchange (20) : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 4bd0783da6fe..31ab69e36d50 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -16,13 +16,13 @@ AdaptiveSparkPlan (72) +- ^ ProjectExecTransformer (35) +- ^ FlushableHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -387,13 +387,13 @@ AdaptiveSparkPlan (120) +- ^ RegularHashAggregateExecTransformer (99) +- ^ RegularHashAggregateExecTransformer (98) +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner (96) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) : +- ^ InputAdapter (90) : +- ^ ShuffleQueryStage (89), Statistics(X) : +- ColumnarExchange (88) : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) : :- ^ InputIteratorTransformer (80) : : +- ^ InputAdapter (79) : : +- ^ ShuffleQueryStage (78), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index ec56c2f99543..7f4249f58548 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (49) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index f7d8f3c3b72f..7ba31590eb06 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -18,7 +18,7 @@ AdaptiveSparkPlan (52) +- ^ RegularHashAggregateExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 4f4492d881fd..379450102787 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (35) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 98249fae7dd7..cb25386ec7a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (42) +- ShuffleQueryStage (25), Statistics(X) +- ColumnarExchange (24) +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index 230e9c890d1d..f6613670b1da 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (64) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 3848cd252b45..d8ec36fbe69f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (57) +- ^ RegularHashAggregateExecTransformer (34) +- ^ RegularHashAggregateExecTransformer (33) +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index 08b902eed5ed..1250303cf3b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (97) +- ^ RegularHashAggregateExecTransformer (61) +- ^ RegularHashAggregateExecTransformer (60) +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) : +- ^ InputAdapter (40) : +- ^ ShuffleQueryStage (39), Statistics(X) : +- ColumnarExchange (38) : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,7 +25,7 @@ AdaptiveSparkPlan (97) : +- ^ ShuffleQueryStage (32), Statistics(X) : +- ColumnarExchange (31) : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) : :- ^ InputIteratorTransformer (16) : : +- ^ InputAdapter (15) : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -43,7 +43,7 @@ AdaptiveSparkPlan (97) : +- ^ ProjectExecTransformer (19) : +- ^ FlushableHashAggregateExecTransformer (18) : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) : +- ^ InputAdapter (48) : +- ^ ShuffleQueryStage (47), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index 29b5d69fa0e8..72aafd6a63af 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -5,7 +5,7 @@ AdaptiveSparkPlan (34) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index c5dc33a694ee..229e17b97d3f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (123) +- ShuffleQueryStage (81), Statistics(X) +- ColumnarExchange (80) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,13 +25,13 @@ AdaptiveSparkPlan (123) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) : :- ^ InputIteratorTransformer (31) : : +- ^ InputAdapter (30) : : +- ^ ShuffleQueryStage (29), Statistics(X) : : +- ColumnarExchange (28) : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) : : :- ^ InputIteratorTransformer (16) : : : +- ^ InputAdapter (15) : : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -55,7 +55,7 @@ AdaptiveSparkPlan (123) : +- ^ ProjectExecTransformer (47) : +- ^ RegularHashAggregateExecTransformer (46) : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ InputIteratorTransformer (39) : : +- ^ InputAdapter (38) : : +- ^ ShuffleQueryStage (37), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index 91bb400aac04..924e087f8ac2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (118) +- ^ ProjectExecTransformer (73) +- ^ FlushableHashAggregateExecTransformer (72) +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) :- ^ InputIteratorTransformer (61) : +- ^ InputAdapter (60) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) : :- ^ InputIteratorTransformer (46) : : +- ^ InputAdapter (45) : : +- ^ ShuffleQueryStage (44), Statistics(X) : : +- ColumnarExchange (43) : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -35,8 +35,8 @@ AdaptiveSparkPlan (118) : : +- ^ ShuffleQueryStage (37), Statistics(X) : : +- ColumnarExchange (36) : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) : : : :- ^ InputIteratorTransformer (16) : : : : +- ^ InputAdapter (15) : : : : +- ^ ShuffleQueryStage (14), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index ba18df0d0119..a40eb7e2b0c2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (46) +- ^ ProjectExecTransformer (19) +- ^ FlushableHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 08588f64d24f..4c1028c2e089 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -7,13 +7,13 @@ AdaptiveSparkPlan (59) +- ^ RegularHashAggregateExecTransformer (35) +- ^ RegularHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 421f6a412ec7..b0cd269ae184 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 50173820058c..082197eda475 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (134) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index 67f4274b3aa3..60738454f20a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (128) +- ^ ProjectExecTransformer (76) +- ^ FlushableHashAggregateExecTransformer (75) +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 16838a0513b3..5549da0cecd2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -16,43 +16,43 @@ AdaptiveSparkPlan (177) +- ^ ProjectExecTransformer (110) +- ^ FlushableHashAggregateExecTransformer (109) +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) :- ^ InputIteratorTransformer (98) : +- ^ InputAdapter (97) : +- ^ ShuffleQueryStage (96), Statistics(X) : +- ColumnarExchange (95) : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) : :- ^ InputIteratorTransformer (83) : : +- ^ InputAdapter (82) : : +- ^ ShuffleQueryStage (81), Statistics(X) : : +- ColumnarExchange (80) : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) : : :- ^ InputIteratorTransformer (68) : : : +- ^ InputAdapter (67) : : : +- ^ ShuffleQueryStage (66), Statistics(X) : : : +- ColumnarExchange (65) : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : : : :- ^ InputIteratorTransformer (53) : : : : +- ^ InputAdapter (52) : : : : +- ^ ShuffleQueryStage (51), Statistics(X) : : : : +- ColumnarExchange (50) : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : : : :- ^ InputIteratorTransformer (38) : : : : : +- ^ InputAdapter (37) : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : : : +- ColumnarExchange (35) : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : : : :- ^ InputIteratorTransformer (23) : : : : : : +- ^ InputAdapter (22) : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : : : +- ColumnarExchange (20) : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index a24b228c5f77..2925cced3f2e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (133) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 79ff600fa6b7..6c98f7aba99d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -12,19 +12,19 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) :- ^ InputIteratorTransformer (38) : +- ^ InputAdapter (37) : +- ^ ShuffleQueryStage (36), Statistics(X) : +- ColumnarExchange (35) : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : :- ^ InputIteratorTransformer (23) : : +- ^ InputAdapter (22) : : +- ^ ShuffleQueryStage (21), Statistics(X) : : +- ColumnarExchange (20) : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index 1b5a8743db5a..59eee0048de4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -16,13 +16,13 @@ AdaptiveSparkPlan (72) +- ^ ProjectExecTransformer (35) +- ^ FlushableHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -391,13 +391,13 @@ AdaptiveSparkPlan (120) +- ^ RegularHashAggregateExecTransformer (99) +- ^ RegularHashAggregateExecTransformer (98) +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner (96) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) : +- ^ InputAdapter (90) : +- ^ ShuffleQueryStage (89), Statistics(X) : +- ColumnarExchange (88) : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) : :- ^ InputIteratorTransformer (80) : : +- ^ InputAdapter (79) : : +- ^ ShuffleQueryStage (78), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index 595d52af5256..1e676f1c8275 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (49) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 10a7818f6c4d..87ea62a744f3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -18,7 +18,7 @@ AdaptiveSparkPlan (52) +- ^ RegularHashAggregateExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 4a4159446f40..d118caee8ae4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (35) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index 90558947bb35..6b5d089f4580 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (42) +- ShuffleQueryStage (25), Statistics(X) +- ColumnarExchange (24) +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 98c7da22a549..57a1fcfd1bce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (64) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index 97c56c7dae8d..03afc1f85103 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (57) +- ^ RegularHashAggregateExecTransformer (34) +- ^ RegularHashAggregateExecTransformer (33) +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index f0537d1fcf07..598b86c342e9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (97) +- ^ RegularHashAggregateExecTransformer (61) +- ^ RegularHashAggregateExecTransformer (60) +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) : +- ^ InputAdapter (40) : +- ^ ShuffleQueryStage (39), Statistics(X) : +- ColumnarExchange (38) : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,7 +25,7 @@ AdaptiveSparkPlan (97) : +- ^ ShuffleQueryStage (32), Statistics(X) : +- ColumnarExchange (31) : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) : :- ^ InputIteratorTransformer (16) : : +- ^ InputAdapter (15) : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -43,7 +43,7 @@ AdaptiveSparkPlan (97) : +- ^ ProjectExecTransformer (19) : +- ^ FlushableHashAggregateExecTransformer (18) : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) : +- ^ InputAdapter (48) : +- ^ ShuffleQueryStage (47), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 7d03f6eca963..2b5d5b20ea54 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -5,7 +5,7 @@ AdaptiveSparkPlan (34) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 4e67d57112e7..ab7fa3fe53c9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (123) +- ShuffleQueryStage (81), Statistics(X) +- ColumnarExchange (80) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,13 +25,13 @@ AdaptiveSparkPlan (123) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) : :- ^ InputIteratorTransformer (31) : : +- ^ InputAdapter (30) : : +- ^ ShuffleQueryStage (29), Statistics(X) : : +- ColumnarExchange (28) : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) : : :- ^ InputIteratorTransformer (16) : : : +- ^ InputAdapter (15) : : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -55,7 +55,7 @@ AdaptiveSparkPlan (123) : +- ^ ProjectExecTransformer (47) : +- ^ RegularHashAggregateExecTransformer (46) : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ InputIteratorTransformer (39) : : +- ^ InputAdapter (38) : : +- ^ ShuffleQueryStage (37), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 658a499a30ce..5c9ac1690ea4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (118) +- ^ ProjectExecTransformer (73) +- ^ FlushableHashAggregateExecTransformer (72) +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) :- ^ InputIteratorTransformer (61) : +- ^ InputAdapter (60) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) : :- ^ InputIteratorTransformer (46) : : +- ^ InputAdapter (45) : : +- ^ ShuffleQueryStage (44), Statistics(X) : : +- ColumnarExchange (43) : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -35,8 +35,8 @@ AdaptiveSparkPlan (118) : : +- ^ ShuffleQueryStage (37), Statistics(X) : : +- ColumnarExchange (36) : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) : : : :- ^ InputIteratorTransformer (16) : : : : +- ^ InputAdapter (15) : : : : +- ^ ShuffleQueryStage (14), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index e94b7b01715f..12dbaf0baf75 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (46) +- ^ ProjectExecTransformer (19) +- ^ FlushableHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index 16615ac0598f..c3de65c763ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -7,13 +7,13 @@ AdaptiveSparkPlan (59) +- ^ RegularHashAggregateExecTransformer (35) +- ^ RegularHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 1d3f8903f89f..ab8ecadcb532 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 12c4c6add240..3296a70a70f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (134) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 03305d572de7..4a641f05b878 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (128) +- ^ ProjectExecTransformer (76) +- ^ FlushableHashAggregateExecTransformer (75) +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index f0176bc5e011..cf125fb93008 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -16,43 +16,43 @@ AdaptiveSparkPlan (177) +- ^ ProjectExecTransformer (110) +- ^ FlushableHashAggregateExecTransformer (109) +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) :- ^ InputIteratorTransformer (98) : +- ^ InputAdapter (97) : +- ^ ShuffleQueryStage (96), Statistics(X) : +- ColumnarExchange (95) : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) : :- ^ InputIteratorTransformer (83) : : +- ^ InputAdapter (82) : : +- ^ ShuffleQueryStage (81), Statistics(X) : : +- ColumnarExchange (80) : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) : : :- ^ InputIteratorTransformer (68) : : : +- ^ InputAdapter (67) : : : +- ^ ShuffleQueryStage (66), Statistics(X) : : : +- ColumnarExchange (65) : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : : : :- ^ InputIteratorTransformer (53) : : : : +- ^ InputAdapter (52) : : : : +- ^ ShuffleQueryStage (51), Statistics(X) : : : : +- ColumnarExchange (50) : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : : : :- ^ InputIteratorTransformer (38) : : : : : +- ^ InputAdapter (37) : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : : : +- ColumnarExchange (35) : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : : : :- ^ InputIteratorTransformer (23) : : : : : : +- ^ InputAdapter (22) : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : : : +- ColumnarExchange (20) : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 3f5479af2b14..426fb391c048 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (133) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 6b7f5bab6b1c..5c3e1a93c499 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -12,19 +12,19 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) :- ^ InputIteratorTransformer (38) : +- ^ InputAdapter (37) : +- ^ ShuffleQueryStage (36) : +- ColumnarExchange (35) : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : :- ^ InputIteratorTransformer (23) : : +- ^ InputAdapter (22) : : +- ^ ShuffleQueryStage (21) : : +- ColumnarExchange (20) : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index 4fe3c8b62ca9..187df108a32a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -16,13 +16,13 @@ AdaptiveSparkPlan (72) +- ^ ProjectExecTransformer (35) +- ^ FlushableHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index f8bca5a8c389..2df3770552eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (49) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index 14f2b770f996..64fd78ed05c9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -18,7 +18,7 @@ AdaptiveSparkPlan (52) +- ^ RegularHashAggregateExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt index 3be22928e4f2..45951bf4c41e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (35) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index 82b075711ecc..2576137dbd7b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -8,7 +8,7 @@ AdaptiveSparkPlan (45) +- ^ ShuffleQueryStage (25) +- ColumnarExchange (24) +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index 4eff981d78bd..875d74780095 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (64) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index 04f31f2d7a90..e3a6142c740b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (57) +- ^ RegularHashAggregateExecTransformer (34) +- ^ RegularHashAggregateExecTransformer (33) +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index ac59fbf6e10f..b1f06863e2ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (97) +- ^ RegularHashAggregateExecTransformer (61) +- ^ RegularHashAggregateExecTransformer (60) +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) : +- ^ InputAdapter (40) : +- ^ ShuffleQueryStage (39) : +- ColumnarExchange (38) : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) @@ -25,7 +25,7 @@ AdaptiveSparkPlan (97) : +- ^ ShuffleQueryStage (32) : +- ColumnarExchange (31) : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) : :- ^ InputIteratorTransformer (16) : : +- ^ InputAdapter (15) : : +- ^ ShuffleQueryStage (14) @@ -43,7 +43,7 @@ AdaptiveSparkPlan (97) : +- ^ ProjectExecTransformer (19) : +- ^ FlushableHashAggregateExecTransformer (18) : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) : +- ^ InputAdapter (48) : +- ^ ShuffleQueryStage (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt index 4bff7da9b473..fe7cc6d3a2e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt @@ -5,7 +5,7 @@ AdaptiveSparkPlan (34) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt index 15ea12dd248e..6e2327ac1336 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt @@ -8,13 +8,13 @@ AdaptiveSparkPlan (126) +- ^ ShuffleQueryStage (81) +- ColumnarExchange (80) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) @@ -27,13 +27,13 @@ AdaptiveSparkPlan (126) : +- ^ ShuffleQueryStage (59) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) : :- ^ InputIteratorTransformer (31) : : +- ^ InputAdapter (30) : : +- ^ ShuffleQueryStage (29) : : +- ColumnarExchange (28) : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) : : :- ^ InputIteratorTransformer (16) : : : +- ^ InputAdapter (15) : : : +- ^ ShuffleQueryStage (14) @@ -57,7 +57,7 @@ AdaptiveSparkPlan (126) : +- ^ ProjectExecTransformer (47) : +- ^ RegularHashAggregateExecTransformer (46) : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ InputIteratorTransformer (39) : : +- ^ InputAdapter (38) : : +- ^ ShuffleQueryStage (37) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index 17fad29860df..0a51e3da621c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -11,19 +11,19 @@ AdaptiveSparkPlan (119) +- ^ ProjectExecTransformer (73) +- ^ FlushableHashAggregateExecTransformer (72) +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) :- ^ InputIteratorTransformer (61) : +- ^ InputAdapter (60) : +- ^ ShuffleQueryStage (59) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) : :- ^ InputIteratorTransformer (46) : : +- ^ InputAdapter (45) : : +- ^ ShuffleQueryStage (44) : : +- ColumnarExchange (43) : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6) @@ -36,8 +36,8 @@ AdaptiveSparkPlan (119) : : +- ^ ShuffleQueryStage (37) : : +- ColumnarExchange (36) : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) : : : :- ^ InputIteratorTransformer (16) : : : : +- ^ InputAdapter (15) : : : : +- ^ ShuffleQueryStage (14) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index 918335dff38b..bc7ca6a0ae16 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (46) +- ^ ProjectExecTransformer (19) +- ^ FlushableHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index 157e13d105e4..80da568524e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -7,13 +7,13 @@ AdaptiveSparkPlan (59) +- ^ RegularHashAggregateExecTransformer (35) +- ^ RegularHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index 74fca4208dd5..02f119e952b1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index c01060f99d4b..d8b833813def 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (134) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index 9cf916718039..77f586cd8897 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (128) +- ^ ProjectExecTransformer (76) +- ^ FlushableHashAggregateExecTransformer (75) +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index 1e191ae4d23c..845d1ac43a77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -16,43 +16,43 @@ AdaptiveSparkPlan (177) +- ^ ProjectExecTransformer (110) +- ^ FlushableHashAggregateExecTransformer (109) +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) :- ^ InputIteratorTransformer (98) : +- ^ InputAdapter (97) : +- ^ ShuffleQueryStage (96) : +- ColumnarExchange (95) : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) : :- ^ InputIteratorTransformer (83) : : +- ^ InputAdapter (82) : : +- ^ ShuffleQueryStage (81) : : +- ColumnarExchange (80) : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) : : :- ^ InputIteratorTransformer (68) : : : +- ^ InputAdapter (67) : : : +- ^ ShuffleQueryStage (66) : : : +- ColumnarExchange (65) : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : : : :- ^ InputIteratorTransformer (53) : : : : +- ^ InputAdapter (52) : : : : +- ^ ShuffleQueryStage (51) : : : : +- ColumnarExchange (50) : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : : : :- ^ InputIteratorTransformer (38) : : : : : +- ^ InputAdapter (37) : : : : : +- ^ ShuffleQueryStage (36) : : : : : +- ColumnarExchange (35) : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : : : :- ^ InputIteratorTransformer (23) : : : : : : +- ^ InputAdapter (22) : : : : : : +- ^ ShuffleQueryStage (21) : : : : : : +- ColumnarExchange (20) : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index a811b042b603..239cda5088bf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (133) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index ae8759a6b86c..11912c5ca02b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -12,19 +12,19 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) :- ^ InputIteratorTransformer (38) : +- ^ InputAdapter (37) : +- ^ ShuffleQueryStage (36), Statistics(X) : +- ColumnarExchange (35) : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : :- ^ InputIteratorTransformer (23) : : +- ^ InputAdapter (22) : : +- ^ ShuffleQueryStage (21), Statistics(X) : : +- ColumnarExchange (20) : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index 3b6c477c0cf5..6180b1966357 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -16,13 +16,13 @@ AdaptiveSparkPlan (72) +- ^ ProjectExecTransformer (35) +- ^ FlushableHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -387,13 +387,13 @@ AdaptiveSparkPlan (120) +- ^ RegularHashAggregateExecTransformer (99) +- ^ RegularHashAggregateExecTransformer (98) +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner (96) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) : +- ^ InputAdapter (90) : +- ^ ShuffleQueryStage (89), Statistics(X) : +- ColumnarExchange (88) : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) : :- ^ InputIteratorTransformer (80) : : +- ^ InputAdapter (79) : : +- ^ ShuffleQueryStage (78), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index 259ddb368b8d..fad8bed52e6a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (49) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index 3f1309f476d9..c10d12cacb71 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -18,7 +18,7 @@ AdaptiveSparkPlan (52) +- ^ RegularHashAggregateExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt index 68716942cea8..222f6d64a5e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (35) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index a9c9e810dd92..95971f8d36a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (42) +- ShuffleQueryStage (25), Statistics(X) +- ColumnarExchange (24) +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index f4b4bc81f48f..39563ad3bf98 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (64) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 50c9d211b3d3..1342964d7f91 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (57) +- ^ RegularHashAggregateExecTransformer (34) +- ^ RegularHashAggregateExecTransformer (33) +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index a29b40d1e996..d45968ecc14a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (97) +- ^ RegularHashAggregateExecTransformer (61) +- ^ RegularHashAggregateExecTransformer (60) +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) : +- ^ InputAdapter (40) : +- ^ ShuffleQueryStage (39), Statistics(X) : +- ColumnarExchange (38) : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,7 +25,7 @@ AdaptiveSparkPlan (97) : +- ^ ShuffleQueryStage (32), Statistics(X) : +- ColumnarExchange (31) : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) : :- ^ InputIteratorTransformer (16) : : +- ^ InputAdapter (15) : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -43,7 +43,7 @@ AdaptiveSparkPlan (97) : +- ^ ProjectExecTransformer (19) : +- ^ FlushableHashAggregateExecTransformer (18) : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) : +- ^ InputAdapter (48) : +- ^ ShuffleQueryStage (47), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt index 69b8adaee61d..bdb709493df4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt @@ -5,7 +5,7 @@ AdaptiveSparkPlan (34) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt index 4fa43644b568..72107db4e377 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (123) +- ShuffleQueryStage (81), Statistics(X) +- ColumnarExchange (80) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,13 +25,13 @@ AdaptiveSparkPlan (123) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) : :- ^ InputIteratorTransformer (31) : : +- ^ InputAdapter (30) : : +- ^ ShuffleQueryStage (29), Statistics(X) : : +- ColumnarExchange (28) : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) : : :- ^ InputIteratorTransformer (16) : : : +- ^ InputAdapter (15) : : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -55,7 +55,7 @@ AdaptiveSparkPlan (123) : +- ^ ProjectExecTransformer (47) : +- ^ RegularHashAggregateExecTransformer (46) : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ InputIteratorTransformer (39) : : +- ^ InputAdapter (38) : : +- ^ ShuffleQueryStage (37), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index f17e6fa164a8..b53c6c158fe9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (118) +- ^ ProjectExecTransformer (73) +- ^ FlushableHashAggregateExecTransformer (72) +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) :- ^ InputIteratorTransformer (61) : +- ^ InputAdapter (60) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) : :- ^ InputIteratorTransformer (46) : : +- ^ InputAdapter (45) : : +- ^ ShuffleQueryStage (44), Statistics(X) : : +- ColumnarExchange (43) : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -35,8 +35,8 @@ AdaptiveSparkPlan (118) : : +- ^ ShuffleQueryStage (37), Statistics(X) : : +- ColumnarExchange (36) : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) : : : :- ^ InputIteratorTransformer (16) : : : : +- ^ InputAdapter (15) : : : : +- ^ ShuffleQueryStage (14), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index f8ce5b6c7485..c63d8516f6a6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (46) +- ^ ProjectExecTransformer (19) +- ^ FlushableHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index 8492e97cdbcc..9461041ef2b0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -7,13 +7,13 @@ AdaptiveSparkPlan (59) +- ^ RegularHashAggregateExecTransformer (35) +- ^ RegularHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index 22e9480fffc8..3c4e85011fd3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 59273229a1ae..677e44e3e6a8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (134) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index 2aa77120c693..8a8f2442f25c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (128) +- ^ ProjectExecTransformer (76) +- ^ FlushableHashAggregateExecTransformer (75) +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index ab8ef221e503..2e6e8038f633 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -16,43 +16,43 @@ AdaptiveSparkPlan (177) +- ^ ProjectExecTransformer (110) +- ^ FlushableHashAggregateExecTransformer (109) +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) :- ^ InputIteratorTransformer (98) : +- ^ InputAdapter (97) : +- ^ ShuffleQueryStage (96), Statistics(X) : +- ColumnarExchange (95) : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) : :- ^ InputIteratorTransformer (83) : : +- ^ InputAdapter (82) : : +- ^ ShuffleQueryStage (81), Statistics(X) : : +- ColumnarExchange (80) : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) : : :- ^ InputIteratorTransformer (68) : : : +- ^ InputAdapter (67) : : : +- ^ ShuffleQueryStage (66), Statistics(X) : : : +- ColumnarExchange (65) : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : : : :- ^ InputIteratorTransformer (53) : : : : +- ^ InputAdapter (52) : : : : +- ^ ShuffleQueryStage (51), Statistics(X) : : : : +- ColumnarExchange (50) : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : : : :- ^ InputIteratorTransformer (38) : : : : : +- ^ InputAdapter (37) : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : : : +- ColumnarExchange (35) : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : : : :- ^ InputIteratorTransformer (23) : : : : : : +- ^ InputAdapter (22) : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : : : +- ColumnarExchange (20) : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 08e6c30a8da3..48b91754df6c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (133) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 873710be9a02..869bb7de0e36 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -12,19 +12,19 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (50) +- ^ FlushableHashAggregateExecTransformer (49) +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) :- ^ InputIteratorTransformer (38) : +- ^ InputAdapter (37) : +- ^ ShuffleQueryStage (36), Statistics(X) : +- ColumnarExchange (35) : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : :- ^ InputIteratorTransformer (23) : : +- ^ InputAdapter (22) : : +- ^ ShuffleQueryStage (21), Statistics(X) : : +- ColumnarExchange (20) : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index bfe059a11c11..494978555e23 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -16,13 +16,13 @@ AdaptiveSparkPlan (72) +- ^ ProjectExecTransformer (35) +- ^ FlushableHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -391,13 +391,13 @@ AdaptiveSparkPlan (120) +- ^ RegularHashAggregateExecTransformer (99) +- ^ RegularHashAggregateExecTransformer (98) +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner (96) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) : +- ^ InputAdapter (90) : +- ^ ShuffleQueryStage (89), Statistics(X) : +- ColumnarExchange (88) : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) : :- ^ InputIteratorTransformer (80) : : +- ^ InputAdapter (79) : : +- ^ ShuffleQueryStage (78), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 85c06695e147..6e4c7befbe03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (49) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index c3526c707e4b..ab591d0dceee 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -18,7 +18,7 @@ AdaptiveSparkPlan (52) +- ^ RegularHashAggregateExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt index fd8974a5b6cb..76297b4084f8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (35) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index 8fd855a59c42..88f14fb60ce6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -6,7 +6,7 @@ AdaptiveSparkPlan (42) +- ShuffleQueryStage (25), Statistics(X) +- ColumnarExchange (24) +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index 498c1e83e15b..cc862d451070 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -22,7 +22,7 @@ AdaptiveSparkPlan (64) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index f15f42b69cd8..a49104ee6fc6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (57) +- ^ RegularHashAggregateExecTransformer (34) +- ^ RegularHashAggregateExecTransformer (33) +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index ad97ed4013f0..c599a4c246b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (97) +- ^ RegularHashAggregateExecTransformer (61) +- ^ RegularHashAggregateExecTransformer (60) +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) : +- ^ InputAdapter (40) : +- ^ ShuffleQueryStage (39), Statistics(X) : +- ColumnarExchange (38) : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,7 +25,7 @@ AdaptiveSparkPlan (97) : +- ^ ShuffleQueryStage (32), Statistics(X) : +- ColumnarExchange (31) : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) : :- ^ InputIteratorTransformer (16) : : +- ^ InputAdapter (15) : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -43,7 +43,7 @@ AdaptiveSparkPlan (97) : +- ^ ProjectExecTransformer (19) : +- ^ FlushableHashAggregateExecTransformer (18) : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) : +- ^ InputAdapter (48) : +- ^ ShuffleQueryStage (47), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt index 4b5c20d3a1b1..214230b1bba6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt @@ -5,7 +5,7 @@ AdaptiveSparkPlan (34) +- ^ RegularHashAggregateExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt index 17f48f8ffa72..85eafca643b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt @@ -6,13 +6,13 @@ AdaptiveSparkPlan (123) +- ShuffleQueryStage (81), Statistics(X) +- ColumnarExchange (80) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -25,13 +25,13 @@ AdaptiveSparkPlan (123) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) : :- ^ InputIteratorTransformer (31) : : +- ^ InputAdapter (30) : : +- ^ ShuffleQueryStage (29), Statistics(X) : : +- ColumnarExchange (28) : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) : : :- ^ InputIteratorTransformer (16) : : : +- ^ InputAdapter (15) : : : +- ^ ShuffleQueryStage (14), Statistics(X) @@ -55,7 +55,7 @@ AdaptiveSparkPlan (123) : +- ^ ProjectExecTransformer (47) : +- ^ RegularHashAggregateExecTransformer (46) : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) : :- ^ InputIteratorTransformer (39) : : +- ^ InputAdapter (38) : : +- ^ ShuffleQueryStage (37), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index 7b5d71ec6d26..04a1a76967b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -10,19 +10,19 @@ AdaptiveSparkPlan (118) +- ^ ProjectExecTransformer (73) +- ^ FlushableHashAggregateExecTransformer (72) +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) :- ^ InputIteratorTransformer (61) : +- ^ InputAdapter (60) : +- ^ ShuffleQueryStage (59), Statistics(X) : +- ColumnarExchange (58) : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) : :- ^ InputIteratorTransformer (46) : : +- ^ InputAdapter (45) : : +- ^ ShuffleQueryStage (44), Statistics(X) : : +- ColumnarExchange (43) : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) : : :- ^ InputIteratorTransformer (8) : : : +- ^ InputAdapter (7) : : : +- ^ ShuffleQueryStage (6), Statistics(X) @@ -35,8 +35,8 @@ AdaptiveSparkPlan (118) : : +- ^ ShuffleQueryStage (37), Statistics(X) : : +- ColumnarExchange (36) : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) : : : :- ^ InputIteratorTransformer (16) : : : : +- ^ InputAdapter (15) : : : : +- ^ ShuffleQueryStage (14), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index 86f293b4ff07..864c9a3d40a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (46) +- ^ ProjectExecTransformer (19) +- ^ FlushableHashAggregateExecTransformer (18) +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index 0c75b0257aba..d25da8196eb2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -7,13 +7,13 @@ AdaptiveSparkPlan (59) +- ^ RegularHashAggregateExecTransformer (35) +- ^ RegularHashAggregateExecTransformer (34) +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) : +- ^ InputAdapter (22) : +- ^ ShuffleQueryStage (21), Statistics(X) : +- ColumnarExchange (20) : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : :- ^ InputIteratorTransformer (8) : : +- ^ InputAdapter (7) : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index ee47fca61115..43af17cfcc73 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -15,7 +15,7 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (20) +- ^ FlushableHashAggregateExecTransformer (19) +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) :- ^ InputIteratorTransformer (8) : +- ^ InputAdapter (7) : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index 39a8f483f639..dd2a2cc31a75 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (134) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index 29c3f048b7f8..32199dcf8e1b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (128) +- ^ ProjectExecTransformer (76) +- ^ FlushableHashAggregateExecTransformer (75) +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index 9c24e9ec5a01..e30d96500f74 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -16,43 +16,43 @@ AdaptiveSparkPlan (177) +- ^ ProjectExecTransformer (110) +- ^ FlushableHashAggregateExecTransformer (109) +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) :- ^ InputIteratorTransformer (98) : +- ^ InputAdapter (97) : +- ^ ShuffleQueryStage (96), Statistics(X) : +- ColumnarExchange (95) : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) : :- ^ InputIteratorTransformer (83) : : +- ^ InputAdapter (82) : : +- ^ ShuffleQueryStage (81), Statistics(X) : : +- ColumnarExchange (80) : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) : : :- ^ InputIteratorTransformer (68) : : : +- ^ InputAdapter (67) : : : +- ^ ShuffleQueryStage (66), Statistics(X) : : : +- ColumnarExchange (65) : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : : : :- ^ InputIteratorTransformer (53) : : : : +- ^ InputAdapter (52) : : : : +- ^ ShuffleQueryStage (51), Statistics(X) : : : : +- ColumnarExchange (50) : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : : : :- ^ InputIteratorTransformer (38) : : : : : +- ^ InputAdapter (37) : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : : : +- ColumnarExchange (35) : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : : : :- ^ InputIteratorTransformer (23) : : : : : : +- ^ InputAdapter (22) : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : : : +- ColumnarExchange (20) : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : : : :- ^ InputIteratorTransformer (8) : : : : : : : +- ^ InputAdapter (7) : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index aef854c3549e..4f61f99709bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -15,31 +15,31 @@ AdaptiveSparkPlan (133) +- ^ ProjectExecTransformer (80) +- ^ FlushableHashAggregateExecTransformer (79) +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) : +- ^ InputAdapter (67) : +- ^ ShuffleQueryStage (66), Statistics(X) : +- ColumnarExchange (65) : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) : :- ^ InputIteratorTransformer (53) : : +- ^ InputAdapter (52) : : +- ^ ShuffleQueryStage (51), Statistics(X) : : +- ColumnarExchange (50) : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) : : :- ^ InputIteratorTransformer (38) : : : +- ^ InputAdapter (37) : : : +- ^ ShuffleQueryStage (36), Statistics(X) : : : +- ColumnarExchange (35) : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) : : : :- ^ InputIteratorTransformer (23) : : : : +- ^ InputAdapter (22) : : : : +- ^ ShuffleQueryStage (21), Statistics(X) : : : : +- ColumnarExchange (20) : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) : : : : :- ^ InputIteratorTransformer (8) : : : : : +- ^ InputAdapter (7) : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala index 6c707e5aa974..cd22c578594c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.physical._ -import org.apache.spark.sql.execution.{ExpandOutputPartitioningShim, SparkPlan} +import org.apache.spark.sql.execution.{ExpandOutputPartitioningShim, ExplainUtils, SparkPlan} import org.apache.spark.sql.execution.joins.{BaseJoinExec, HashedRelationBroadcastMode, HashJoin} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types._ @@ -100,6 +100,11 @@ trait HashJoinLikeExecTransformer extends BaseJoinExec with TransformSupport { def joinBuildSide: BuildSide def hashJoinType: JoinType + override def simpleStringWithNodeId(): String = { + val opId = ExplainUtils.getOpId(this) + s"$nodeName $joinType $joinBuildSide ($opId)".trim + } + // Note: "metrics" is made transient to avoid sending driver-side metrics to tasks. @transient override lazy val metrics: Map[String, SQLMetric] = BackendsApiManager.getMetricsApiInstance.genHashJoinTransformerMetrics(sparkContext) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala index 98b3666f84ba..f032c4ca0087 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SortMergeJoinExecTransformer.scala @@ -55,11 +55,6 @@ abstract class SortMergeJoinExecTransformerBase( val (bufferedKeys, streamedKeys, bufferedPlan, streamedPlan) = (rightKeys, leftKeys, right, left) - override def simpleStringWithNodeId(): String = { - val opId = ExplainUtils.getOpId(this) - s"$nodeName $joinType ($opId)".trim - } - override def verboseStringWithOperatorId(): String = { val joinCondStr = if (condition.isDefined) { s"${condition.get}" @@ -255,7 +250,6 @@ case class SortMergeJoinExecTransformer( projectList) { override protected def doValidateInternal(): ValidationResult = { - val substraitContext = new SubstraitContext // Firstly, need to check if the Substrait plan for this operator can be successfully generated. if (substraitJoinType == JoinRel.JoinType.JOIN_TYPE_OUTER) { return ValidationResult From 00dda1b67ec81dc841643e1c02323f32df5525d9 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Wed, 5 Jun 2024 10:10:37 +0800 Subject: [PATCH 207/402] [GLUTEN-5959] Fix function replace report an error with null value (#5960) What changes were proposed in this pull request? Kyligence/ClickHouse#493 Fix replace null value (Fixes: #5959) How was this patch tested? test by ut --- ...GlutenClickhouseStringFunctionsSuite.scala | 100 ++++++++++++++++++ .../GlutenFunctionValidateSuite.scala | 37 ------- cpp-ch/clickhouse.version | 2 +- 3 files changed, 101 insertions(+), 38 deletions(-) create mode 100644 backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala new file mode 100644 index 000000000000..029a763c983c --- /dev/null +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.SparkConf + +class GlutenClickhouseStringFunctionsSuite extends GlutenClickHouseWholeStageTransformerSuite { + + /** Run Gluten + ClickHouse Backend with SortShuffleManager */ + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "sort") + .set("spark.io.compression.codec", "SNAPPY") + .set("spark.sql.shuffle.partitions", "5") + .set("spark.sql.autoBroadcastJoinThreshold", "10MB") + } + + test("GLUTEN-5821: trim_character support value from column.") { + withTable("trim") { + sql("create table trim(a String, b String) using parquet") + sql(""" + |insert into trim values + | ('aba', 'a'),('bba', 'b'),('abcdef', 'abcd'), (null, '123'),('123', null) + |""".stripMargin) + + val sql_str = + s"""select + | trim(both b from a) + | from trim + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + + test("GLUTEN-5897: fix regexp_extract with bracket") { + withTable("regexp_extract_bracket") { + sql("create table regexp_extract_bracket(a String) using parquet") + sql(""" + |insert into regexp_extract_bracket + | values ('123.123abc-abc'),('123-LOW'),('123]abc-abc') + |""".stripMargin) + + val sql_str = + s"""select + | regexp_extract(a, '([0-9][[\\.][0-9]]*)', 1) + | , regexp_extract(a, '([0-9][[\\.][0-9]]*)', 1) + | , regexp_extract(a, '([0-9][[]]]*)', 1) + | from regexp_extract_bracket + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + + test("replace") { + val tableName = "replace_table" + withTable(tableName) { + sql(s"create table $tableName(src String, idx String, dest String) using parquet") + sql(s""" + |insert into $tableName values + | (null, null, null), + | ('1', '1', null), + | ('1', '1', '2'), + | ('1', null, '2'), + | ('1', '1', '3'), + | (null, '1', '2'), + | ('1', '', '3') + """.stripMargin) + + val sql_str = + s""" + |select + | REPLACE(src, idx, dest), + | REPLACE(src, null, dest), + | REPLACE(null, null, dest), + | REPLACE(null, null, null), + | REPLACE(src, '1', null) + | from $tableName + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + +} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index cfe8ea95abcd..a561fe7cb442 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -707,41 +707,4 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS } } - - test("GLUTEN-5821: trim_character support value from column.") { - withTable("trim") { - sql("create table trim(a String, b String) using parquet") - sql(""" - |insert into trim values ('aba', 'a'),('bba', 'b'),('abcdef', 'abcd') - |""".stripMargin) - - val sql_str = - s"""select - | trim(both b from a) - | from trim - """.stripMargin - - runQueryAndCompare(sql_str) { _ => } - } - } - - test("GLUTEN-5897: fix regexp_extract with bracket") { - withTable("regexp_extract_bracket") { - sql("create table regexp_extract_bracket(a String) using parquet") - sql( - """ - |insert into regexp_extract_bracket values ('123.123abc-abc'),('123-LOW'),('123]abc-abc') - |""".stripMargin) - - val sql_str = - s"""select - | regexp_extract(a, '([0-9][[\\.][0-9]]*)', 1) - | , regexp_extract(a, '([0-9][[\\.][0-9]]*)', 1) - | , regexp_extract(a, '([0-9][[]]]*)', 1) - | from regexp_extract_bracket - """.stripMargin - - runQueryAndCompare(sql_str) { _ => } - } - } } diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 97e9df5a7688..c1baa6037996 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence CH_BRANCH=rebase_ch/20240527 -CH_COMMIT=7ebb938593259aeb09952289ef7553b045ce4c15 +CH_COMMIT=1388dcb5b0bbb630af259280f4287e3342ca6237 From 59aaa1cc36686b34900b44b920679ed335f94302 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 5 Jun 2024 11:41:36 +0800 Subject: [PATCH 208/402] [VL] Quick fix for Uniffle CI error (#5986) --- .../apache/gluten/integration/BaseMixin.java | 2 +- .../org/apache/gluten/integration/Suite.scala | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java index dc1691e50021..93c82a6fa257 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java @@ -75,7 +75,7 @@ public class BaseMixin { @CommandLine.Option(names = {"--shuffle-partitions"}, description = "Shuffle partition number", defaultValue = "100") private int shufflePartitions; - @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value", defaultValue = "100") + @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value. -1 for letting Spark choose an appropriate number.", defaultValue = "-1") private int scanPartitions; @CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark config entries applying to generated Spark session. E.g. --extra-conf=k1=v1 --extra-conf=k2=v2") diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala index bb5cb1889125..51e1777e25f8 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala @@ -103,16 +103,18 @@ abstract class Suite( sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.codegen.wholeStage", "false") } - // Scan partition number. - sessionSwitcher - .defaultConf() - .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}") - sessionSwitcher - .defaultConf() - .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0") - sessionSwitcher - .defaultConf() - .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}") + if (scanPartitions != -1) { + // Scan partition number. + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}") + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0") + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}") + } extraSparkConf.toStream.foreach { kv => sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2) From c9350fb204f9912c7cb5bfa4534b0aeabfac683c Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Wed, 5 Jun 2024 13:44:49 +0800 Subject: [PATCH 209/402] [CORE] ExpandFallbackPolicy should propagate fallback reason to vanilla SparkPlan (#5971) --- .../columnar/ExpandFallbackPolicy.scala | 15 ++++- .../sql/gluten/GlutenFallbackSuite.scala | 65 +++++++++++++++++++ .../sql/gluten/GlutenFallbackSuite.scala | 43 ++++++++++-- .../sql/gluten/GlutenFallbackSuite.scala | 43 ++++++++++-- 4 files changed, 156 insertions(+), 10 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala index 6f8d7cde703b..4ee153173c5c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala @@ -235,7 +235,18 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP } } - private def fallbackToRowBasedPlan(outputsColumnar: Boolean): SparkPlan = { + private def fallbackToRowBasedPlan(glutenPlan: SparkPlan, outputsColumnar: Boolean): SparkPlan = { + // Propagate fallback reason to vanilla SparkPlan + glutenPlan.foreach { + case _: GlutenPlan => + case p: SparkPlan if TransformHints.isNotTransformable(p) && p.logicalLink.isDefined => + originalPlan + .find(_.logicalLink.exists(_.fastEquals(p.logicalLink.get))) + .filterNot(TransformHints.isNotTransformable) + .foreach(origin => TransformHints.tag(origin, TransformHints.getHint(p))) + case _ => + } + val planWithTransitions = Transitions.insertTransitions(originalPlan, outputsColumnar) planWithTransitions } @@ -259,7 +270,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP // Scan Parquet // | // ColumnarToRow - val vanillaSparkPlan = fallbackToRowBasedPlan(outputsColumnar) + val vanillaSparkPlan = fallbackToRowBasedPlan(plan, outputsColumnar) val vanillaSparkTransitionCost = countTransitionCostForVanillaSparkPlan(vanillaSparkPlan) if ( GlutenConfig.getConf.fallbackPreferColumnar && diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala index b85dd6a3518e..6860d6a12958 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala @@ -23,7 +23,9 @@ import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} import org.apache.spark.sql.{GlutenSQLTestsTrait, Row} +import org.apache.spark.sql.execution.ProjectExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.ui.{GlutenSQLAppStatusStore, SparkListenerSQLExecutionStart} import org.apache.spark.status.ElementTrackingStore @@ -161,4 +163,67 @@ class GlutenFallbackSuite extends GlutenSQLTestsTrait with AdaptiveSparkPlanHelp } } } + + test("Add logical link to rewritten spark plan") { + val events = new ArrayBuffer[GlutenPlanFallbackEvent] + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: GlutenPlanFallbackEvent => events.append(e) + case _ => + } + } + } + spark.sparkContext.addSparkListener(listener) + withSQLConf(GlutenConfig.EXPRESSION_BLACK_LIST.key -> "add") { + try { + val df = spark.sql("select sum(id + 1) from range(10)") + df.collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + val project = find(df.queryExecution.executedPlan) { + _.isInstanceOf[ProjectExec] + } + assert(project.isDefined) + assert( + events.exists(_.fallbackNodeToReason.values.toSet + .exists(_.contains("Not supported to map spark function name")))) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + } + + test("ExpandFallbackPolicy should propagate fallback reason to vanilla SparkPlan") { + val events = new ArrayBuffer[GlutenPlanFallbackEvent] + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: GlutenPlanFallbackEvent => events.append(e) + case _ => + } + } + } + spark.sparkContext.addSparkListener(listener) + spark.range(10).selectExpr("id as c1", "id as c2").write.format("parquet").saveAsTable("t") + withTable("t") { + withSQLConf( + GlutenConfig.EXPRESSION_BLACK_LIST.key -> "max", + GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1") { + try { + val df = spark.sql("select c2, max(c1) as id from t group by c2") + df.collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + val agg = collect(df.queryExecution.executedPlan) { case a: HashAggregateExec => a } + assert(agg.size == 2) + assert( + events.count( + _.fallbackNodeToReason.values.toSet.exists(_.contains( + "Could not find a valid substrait mapping name for max" + ))) == 2) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + } + } } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala index 9e8c7e54291a..fd6aa047558f 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala @@ -25,6 +25,7 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} import org.apache.spark.sql.{GlutenSQLTestsTrait, Row} import org.apache.spark.sql.execution.ProjectExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.ui.{GlutenSQLAppStatusStore, SparkListenerSQLExecutionStart} import org.apache.spark.status.ElementTrackingStore @@ -168,18 +169,52 @@ class GlutenFallbackSuite extends GlutenSQLTestsTrait with AdaptiveSparkPlanHelp withSQLConf(GlutenConfig.EXPRESSION_BLACK_LIST.key -> "add") { try { val df = spark.sql("select sum(id + 1) from range(10)") - spark.sparkContext.listenerBus.waitUntilEmpty() df.collect() + spark.sparkContext.listenerBus.waitUntilEmpty() val project = find(df.queryExecution.executedPlan) { _.isInstanceOf[ProjectExec] } assert(project.isDefined) - events.exists( - _.fallbackNodeToReason.values.toSet - .contains("Project: Not supported to map spark function name")) + assert( + events.exists(_.fallbackNodeToReason.values.toSet + .exists(_.contains("Not supported to map spark function name")))) } finally { spark.sparkContext.removeSparkListener(listener) } } } + + test("ExpandFallbackPolicy should propagate fallback reason to vanilla SparkPlan") { + val events = new ArrayBuffer[GlutenPlanFallbackEvent] + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: GlutenPlanFallbackEvent => events.append(e) + case _ => + } + } + } + spark.sparkContext.addSparkListener(listener) + spark.range(10).selectExpr("id as c1", "id as c2").write.format("parquet").saveAsTable("t") + withTable("t") { + withSQLConf( + GlutenConfig.EXPRESSION_BLACK_LIST.key -> "max", + GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1") { + try { + val df = spark.sql("select c2, max(c1) as id from t group by c2") + df.collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + val agg = collect(df.queryExecution.executedPlan) { case a: HashAggregateExec => a } + assert(agg.size == 2) + assert( + events.count( + _.fallbackNodeToReason.values.toSet.exists(_.contains( + "Could not find a valid substrait mapping name for max" + ))) == 2) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + } + } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala index 9e8c7e54291a..fd6aa047558f 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala @@ -25,6 +25,7 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} import org.apache.spark.sql.{GlutenSQLTestsTrait, Row} import org.apache.spark.sql.execution.ProjectExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.ui.{GlutenSQLAppStatusStore, SparkListenerSQLExecutionStart} import org.apache.spark.status.ElementTrackingStore @@ -168,18 +169,52 @@ class GlutenFallbackSuite extends GlutenSQLTestsTrait with AdaptiveSparkPlanHelp withSQLConf(GlutenConfig.EXPRESSION_BLACK_LIST.key -> "add") { try { val df = spark.sql("select sum(id + 1) from range(10)") - spark.sparkContext.listenerBus.waitUntilEmpty() df.collect() + spark.sparkContext.listenerBus.waitUntilEmpty() val project = find(df.queryExecution.executedPlan) { _.isInstanceOf[ProjectExec] } assert(project.isDefined) - events.exists( - _.fallbackNodeToReason.values.toSet - .contains("Project: Not supported to map spark function name")) + assert( + events.exists(_.fallbackNodeToReason.values.toSet + .exists(_.contains("Not supported to map spark function name")))) } finally { spark.sparkContext.removeSparkListener(listener) } } } + + test("ExpandFallbackPolicy should propagate fallback reason to vanilla SparkPlan") { + val events = new ArrayBuffer[GlutenPlanFallbackEvent] + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: GlutenPlanFallbackEvent => events.append(e) + case _ => + } + } + } + spark.sparkContext.addSparkListener(listener) + spark.range(10).selectExpr("id as c1", "id as c2").write.format("parquet").saveAsTable("t") + withTable("t") { + withSQLConf( + GlutenConfig.EXPRESSION_BLACK_LIST.key -> "max", + GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1") { + try { + val df = spark.sql("select c2, max(c1) as id from t group by c2") + df.collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + val agg = collect(df.queryExecution.executedPlan) { case a: HashAggregateExec => a } + assert(agg.size == 2) + assert( + events.count( + _.fallbackNodeToReason.values.toSet.exists(_.contains( + "Could not find a valid substrait mapping name for max" + ))) == 2) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + } + } } From 15c7c5808eb26468b9fe0e237d5e5edf26490fa6 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 5 Jun 2024 16:20:58 +0800 Subject: [PATCH 210/402] [VL] Do not skip updating children's metrics while visiting an operator with NoopMetricsUpdater (#5933) --- .../metrics/HashAggregateMetricsUpdater.scala | 2 +- .../metrics/HashJoinMetricsUpdater.scala | 2 +- .../apache/gluten/metrics/MetricsUtil.scala | 6 ++-- .../gluten/execution/TopNTransformer.scala | 4 +-- .../gluten/execution/VeloxMetricsSuite.scala | 36 +++++++++++++++++++ .../execution/WholeStageTransformer.scala | 6 ++-- .../columnar/enumerated/RemoveFilter.scala | 6 ++-- .../gluten/metrics/MetricsUpdater.scala | 31 ++++++++++++---- .../apache/gluten/metrics/MetricsUtil.scala | 36 ++++++++++--------- 9 files changed, 93 insertions(+), 36 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala index e2014e5b8b84..b035d7a04fb0 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala @@ -65,7 +65,7 @@ class HashAggregateMetricsUpdater(val metrics: Map[String, SQLMetric]) } } } catch { - case e: Throwable => + case e: Exception => logError(s"Updating native metrics failed due to ${e.getCause}.") throw e } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashJoinMetricsUpdater.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashJoinMetricsUpdater.scala index 3c35286c1c13..ca891bac27c6 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashJoinMetricsUpdater.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/HashJoinMetricsUpdater.scala @@ -104,7 +104,7 @@ class HashJoinMetricsUpdater(val metrics: Map[String, SQLMetric]) } } } catch { - case e: Throwable => + case e: Exception => logError(s"Updating native metrics failed due to ${e.getCause}.") throw e } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala index a6dfb3dbcb1f..1376dc6a82d1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala @@ -41,7 +41,7 @@ object MetricsUtil extends Logging { case t: TransformSupport => MetricsUpdaterTree(t.metricsUpdater(), t.children.map(treeifyMetricsUpdaters)) case _ => - MetricsUpdaterTree(NoopMetricsUpdater, Seq()) + MetricsUpdaterTree(MetricsUpdater.Terminate, Seq()) } } @@ -107,7 +107,7 @@ object MetricsUtil extends Logging { s"Updating native metrics failed due to the wrong size of metrics data: " + s"$numNativeMetrics") () - } else if (mutNode.updater == NoopMetricsUpdater) { + } else if (mutNode.updater == MetricsUpdater.Terminate) { () } else { updateTransformerMetricsInternal( @@ -159,7 +159,7 @@ object MetricsUtil extends Logging { mutNode.children.foreach { child => - if (child.updater != NoopMetricsUpdater) { + if (child.updater != MetricsUpdater.Terminate) { val result = updateTransformerMetricsInternal( child, relMap, diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala index c2d12415c78b..01c89bee217b 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/TopNTransformer.scala @@ -19,7 +19,7 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.expression.{ConverterUtils, ExpressionConverter} import org.apache.gluten.extension.ValidationResult -import org.apache.gluten.metrics.{MetricsUpdater, NoopMetricsUpdater} +import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.`type`.TypeBuilder import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.extensions.ExtensionBuilder @@ -114,5 +114,5 @@ case class TopNTransformer( } } - override def metricsUpdater(): MetricsUpdater = NoopMetricsUpdater // TODO + override def metricsUpdater(): MetricsUpdater = MetricsUpdater.Todo // TODO } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala index ce8450fea423..468f26259219 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala @@ -19,6 +19,7 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.sql.shims.SparkShimLoader +import org.apache.spark.SparkConf import org.apache.spark.sql.execution.CommandResultExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf @@ -52,6 +53,11 @@ class VeloxMetricsSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa super.afterAll() } + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + } + test("test sort merge join metrics") { withSQLConf( GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key -> "false", @@ -143,6 +149,36 @@ class VeloxMetricsSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } } + test("Metrics of window") { + runQueryAndCompare("SELECT c1, c2, sum(c2) over (partition by c1) as s FROM metrics_t1") { + df => + val window = find(df.queryExecution.executedPlan) { + case _: WindowExecTransformer => true + case _ => false + } + assert(window.isDefined) + val metrics = window.get.metrics + assert(metrics("numOutputRows").value == 100) + assert(metrics("outputVectors").value == 2) + } + } + + test("Metrics of noop filter's children") { + withSQLConf("spark.gluten.ras.enabled" -> "true") { + runQueryAndCompare("SELECT c1, c2 FROM metrics_t1 where c1 < 50") { + df => + val scan = find(df.queryExecution.executedPlan) { + case _: FileSourceScanExecTransformer => true + case _ => false + } + assert(scan.isDefined) + val metrics = scan.get.metrics + assert(metrics("rawInputRows").value == 100) + assert(metrics("outputVectors").value == 1) + } + } + } + test("Write metrics") { if (SparkShimLoader.getSparkVersion.startsWith("3.4")) { withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala index ed691fc09613..7dfa0563d743 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala @@ -21,7 +21,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenException import org.apache.gluten.expression._ import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.metrics.{GlutenTimeMetric, MetricsUpdater, NoopMetricsUpdater} +import org.apache.gluten.metrics.{GlutenTimeMetric, MetricsUpdater} import org.apache.gluten.substrait.`type`.{TypeBuilder, TypeNode} import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.plan.{PlanBuilder, PlanNode} @@ -350,7 +350,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f override def metricsUpdater(): MetricsUpdater = { child match { case transformer: TransformSupport => transformer.metricsUpdater() - case _ => NoopMetricsUpdater + case _ => MetricsUpdater.None } } @@ -361,7 +361,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f case _ => false } .map(_.asInstanceOf[TransformSupport].metricsUpdater()) - .getOrElse(NoopMetricsUpdater) + .getOrElse(MetricsUpdater.None) } override protected def withNewChildInternal(newChild: SparkPlan): WholeStageTransformer = diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index b980c24227d5..5d7209dfbfb4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -17,7 +17,7 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.execution._ -import org.apache.gluten.metrics.{MetricsUpdater, NoopMetricsUpdater} +import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.ras.path.Pattern._ import org.apache.gluten.ras.path.Pattern.Matchers._ import org.apache.gluten.ras.rule.{RasRule, Shape} @@ -54,7 +54,7 @@ object RemoveFilter extends RasRule[SparkPlan] { leaf(clazz(classOf[BasicScanExecTransformer])) ).build()) - // A noop filter placeholder that indicates that all conditions are pushed down to scan. + // A noop filter placeholder that indicates that all conditions were pushed down to scan. // // This operator has zero cost in cost model to avoid planner from choosing the // original filter-scan that doesn't have all conditions pushed down to scan. @@ -71,7 +71,7 @@ object RemoveFilter extends RasRule[SparkPlan] { // spark.sql.adaptive.logLevel=ERROR. case class NoopFilter(override val child: SparkPlan, override val output: Seq[Attribute]) extends UnaryTransformSupport { - override def metricsUpdater(): MetricsUpdater = NoopMetricsUpdater + override def metricsUpdater(): MetricsUpdater = MetricsUpdater.None override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = copy(newChild) override def outputPartitioning: Partitioning = child.outputPartitioning override def outputOrdering: Seq[SortOrder] = child.outputOrdering diff --git a/gluten-core/src/main/scala/org/apache/gluten/metrics/MetricsUpdater.scala b/gluten-core/src/main/scala/org/apache/gluten/metrics/MetricsUpdater.scala index 0a622ba0b37d..5201df3b3472 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/metrics/MetricsUpdater.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/metrics/MetricsUpdater.scala @@ -16,7 +16,6 @@ */ package org.apache.gluten.metrics -import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.utils.OASPackageBridge.InputMetricsWrapper /** @@ -26,16 +25,34 @@ import org.apache.spark.sql.utils.OASPackageBridge.InputMetricsWrapper * TODO: place it to some other where since it's used not only by whole stage facilities */ trait MetricsUpdater extends Serializable { + def updateInputMetrics(inputMetrics: InputMetricsWrapper): Unit = {} + def updateNativeMetrics(operatorMetrics: IOperatorMetrics): Unit = {} +} - def metrics: Map[String, SQLMetric] +object MetricsUpdater { + // An empty metrics updater. Used when the operator generates native metrics but + // it's yet unwanted to update the metrics in JVM side. + object Todo extends MetricsUpdater {} - def updateInputMetrics(inputMetrics: InputMetricsWrapper): Unit = {} + // Used when the operator doesn't generate native metrics. It could be because + // the operator doesn't generate any native query plan. + object None extends MetricsUpdater { + override def updateInputMetrics(inputMetrics: InputMetricsWrapper): Unit = + throw new UnsupportedOperationException() + override def updateNativeMetrics(operatorMetrics: IOperatorMetrics): Unit = + throw new UnsupportedOperationException() + } - def updateNativeMetrics(operatorMetrics: IOperatorMetrics): Unit = {} + // Indicates a branch of a MetricsUpdaterTree is terminated. It's not bound to + // any operators. + object Terminate extends MetricsUpdater { + override def updateInputMetrics(inputMetrics: InputMetricsWrapper): Unit = + throw new UnsupportedOperationException() + override def updateNativeMetrics(operatorMetrics: IOperatorMetrics): Unit = + throw new UnsupportedOperationException() + } } final case class MetricsUpdaterTree(updater: MetricsUpdater, children: Seq[MetricsUpdaterTree]) -object NoopMetricsUpdater extends MetricsUpdater { - override def metrics: Map[String, SQLMetric] = Map.empty -} +object MetricsUpdaterTree {} diff --git a/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala b/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala index f11800b89c31..0c387b429212 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala @@ -54,10 +54,13 @@ object MetricsUtil extends Logging { MetricsUpdaterTree( smj.metricsUpdater(), Seq(treeifyMetricsUpdaters(smj.bufferedPlan), treeifyMetricsUpdaters(smj.streamedPlan))) + case t: TransformSupport if t.metricsUpdater() == MetricsUpdater.None => + assert(t.children.size == 1, "MetricsUpdater.None can only be used on unary operator") + treeifyMetricsUpdaters(t.children.head) case t: TransformSupport => MetricsUpdaterTree(t.metricsUpdater(), t.children.map(treeifyMetricsUpdaters)) case _ => - MetricsUpdaterTree(NoopMetricsUpdater, Seq()) + MetricsUpdaterTree(MetricsUpdater.Terminate, Seq()) } } @@ -180,6 +183,8 @@ object MetricsUtil extends Logging { ) } + // FIXME: Metrics updating code is too magical to maintain. Tree-walking algorithm should be made + // more declarative than by counting down these counters that don't have fixed definition. /** * @return * operator index and metrics index @@ -192,6 +197,9 @@ object MetricsUtil extends Logging { metricsIdx: Int, joinParamsMap: JMap[JLong, JoinParams], aggParamsMap: JMap[JLong, AggregationParams]): (JLong, Int) = { + if (mutNode.updater == MetricsUpdater.Terminate) { + return (operatorIdx, metricsIdx) + } val operatorMetrics = new JArrayList[OperatorMetrics]() var curMetricsIdx = metricsIdx relMap @@ -245,18 +253,16 @@ object MetricsUtil extends Logging { mutNode.children.foreach { child => - if (child.updater != NoopMetricsUpdater) { - val result = updateTransformerMetricsInternal( - child, - relMap, - newOperatorIdx, - metrics, - newMetricsIdx, - joinParamsMap, - aggParamsMap) - newOperatorIdx = result._1 - newMetricsIdx = result._2 - } + val result = updateTransformerMetricsInternal( + child, + relMap, + newOperatorIdx, + metrics, + newMetricsIdx, + joinParamsMap, + aggParamsMap) + newOperatorIdx = result._1 + newMetricsIdx = result._2 } (newOperatorIdx, newMetricsIdx) @@ -292,8 +298,6 @@ object MetricsUtil extends Logging { val numNativeMetrics = metrics.inputRows.length if (numNativeMetrics == 0) { () - } else if (mutNode.updater == NoopMetricsUpdater) { - () } else { updateTransformerMetricsInternal( mutNode, @@ -305,7 +309,7 @@ object MetricsUtil extends Logging { aggParamsMap) } } catch { - case e: Throwable => + case e: Exception => logWarning(s"Updating native metrics failed due to ${e.getCause}.") () } From a3dff0491b9beb489a03efe6f4ac566ed003300e Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 5 Jun 2024 17:28:40 +0800 Subject: [PATCH 211/402] [VL] Add unknown type to shuffle cpp ut (#5973) --- cpp/velox/tests/VeloxShuffleWriterTest.cc | 10 ++++------ cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cpp/velox/tests/VeloxShuffleWriterTest.cc b/cpp/velox/tests/VeloxShuffleWriterTest.cc index 52649128a999..1c1be6fc1b6f 100644 --- a/cpp/velox/tests/VeloxShuffleWriterTest.cc +++ b/cpp/velox/tests/VeloxShuffleWriterTest.cc @@ -375,6 +375,9 @@ TEST_P(RoundRobinPartitioningShuffleWriter, preAllocForceRealloc) { } TEST_P(RoundRobinPartitioningShuffleWriter, preAllocForceReuse) { + if (GetParam().shuffleWriterType == kSortShuffle) { + return; + } ASSERT_NOT_OK(initShuffleWriterOptions()); shuffleWriterOptions_.bufferReallocThreshold = 1; // Force re-alloc on buffer size changed. auto shuffleWriter = createShuffleWriter(defaultArrowMemoryPool().get()); @@ -440,12 +443,7 @@ TEST_P(RoundRobinPartitioningShuffleWriter, spillVerifyResult) { auto blockPid2 = takeRows({inputVector1_}, {{1, 3, 5, 7, 9, 1, 3, 5, 7, 9, 1, 3, 5, 7, 9}}); // Stop and verify. - shuffleWriteReadMultiBlocks( - *shuffleWriter, - 2, - inputVector1_->type(), - // {{block1Pid1, block1Pid1, block1Pid1}, {block1Pid2, block1Pid2, block1Pid2}}); - {{blockPid1}, {blockPid2}}); + shuffleWriteReadMultiBlocks(*shuffleWriter, 2, inputVector1_->type(), {{blockPid1}, {blockPid2}}); } TEST_F(VeloxShuffleWriterMemoryTest, memoryLeak) { diff --git a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h index 94e2b071b430..fd3ae3d547fe 100644 --- a/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h +++ b/cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h @@ -26,6 +26,7 @@ #include "shuffle/PartitionWriter.h" #include "shuffle/VeloxShuffleReader.h" #include "utils/Compression.h" +#include "velox/type/Type.h" #include "velox/vector/tests/VectorTestUtils.h" namespace gluten { @@ -119,7 +120,7 @@ class VeloxShuffleWriterTestBase : public facebook::velox::test::VectorTestBase {"alice0", "bob1", "alice2", "bob3", "Alice4", "Bob5", "AlicE6", "boB7", "ALICE8", "BOB9"}), makeNullableFlatVector( {"alice", "bob", std::nullopt, std::nullopt, "Alice", "Bob", std::nullopt, "alicE", std::nullopt, "boB"}), - }; + facebook::velox::BaseVector::create(facebook::velox::UNKNOWN(), 10, pool())}; children2_ = { makeNullableFlatVector({std::nullopt, std::nullopt}), @@ -132,7 +133,7 @@ class VeloxShuffleWriterTestBase : public facebook::velox::test::VectorTestBase {"bob", "alicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealicealice"}), makeNullableFlatVector({std::nullopt, std::nullopt}), - }; + facebook::velox::BaseVector::create(facebook::velox::UNKNOWN(), 2, pool())}; childrenNoNull_ = { makeFlatVector({0, 1}), From 9ea1618498372a1e7a78b920451453165b427b1a Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 5 Jun 2024 18:08:34 +0800 Subject: [PATCH 212/402] [VL] Daily Update Velox Version (2024_06_04) (#5968) Co-authored-by: PHILO-HE --- cpp/velox/compute/VeloxBackend.cc | 4 ++-- dev/ci-velox-buildstatic.sh | 2 +- ep/build-velox/src/get_velox.sh | 2 +- ep/build-velox/src/modify_velox.patch | 26 ++++++++++++-------------- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 187c36e1e8bd..10d1c7529710 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -156,8 +156,8 @@ void VeloxBackend::initCache() { cacheFilePrefix_ = getCacheFilePrefix(); std::string ssdCachePath = ssdCachePathPrefix + "/" + cacheFilePrefix_; ssdCacheExecutor_ = std::make_unique(ssdCacheIOThreads); - auto ssd = - std::make_unique(ssdCachePath, ssdCacheSize, ssdCacheShards, ssdCacheExecutor_.get()); + const cache::SsdCache::Config config(ssdCachePath, ssdCacheSize, ssdCacheShards, ssdCacheExecutor_.get()); + auto ssd = std::make_unique(config); std::error_code ec; const std::filesystem::space_info si = std::filesystem::space(ssdCachePathPrefix, ec); diff --git a/dev/ci-velox-buildstatic.sh b/dev/ci-velox-buildstatic.sh index 227bad36053b..74688ff307cb 100755 --- a/dev/ci-velox-buildstatic.sh +++ b/dev/ci-velox-buildstatic.sh @@ -6,4 +6,4 @@ cd $GITHUB_WORKSPACE/ source ./dev/vcpkg/env.sh sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt export NUM_THREADS=4 -./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON +./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=ON diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index e1133e88c717..5aa3f2b379c0 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_03 +VELOX_BRANCH=2024_06_04 VELOX_HOME="" #Set on run gluten on HDFS diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch index 09af35020842..81560917d620 100644 --- a/ep/build-velox/src/modify_velox.patch +++ b/ep/build-velox/src/modify_velox.patch @@ -36,10 +36,10 @@ index d49115f12..1aaa8e532 100644 + endif() endif() diff --git a/CMakeLists.txt b/CMakeLists.txt -index 53aaf4391..90aba6916 100644 +index 5c7bf770a..9f897f577 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -243,10 +243,15 @@ if(VELOX_ENABLE_ABFS) +@@ -234,10 +234,15 @@ if(VELOX_ENABLE_ABFS) endif() if(VELOX_ENABLE_HDFS) @@ -59,7 +59,7 @@ index 53aaf4391..90aba6916 100644 add_definitions(-DVELOX_ENABLE_HDFS3) endif() -@@ -386,7 +391,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES}) +@@ -377,7 +382,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES}) # for reference. find_package(range-v3) set_source(gflags) @@ -68,6 +68,15 @@ index 53aaf4391..90aba6916 100644 if(NOT TARGET gflags::gflags) # This is a bit convoluted, but we want to be able to use gflags::gflags as a # target even when velox is built as a subproject which uses +@@ -441,7 +446,7 @@ if(${VELOX_BUILD_MINIMAL_WITH_DWIO} + + # Locate or build protobuf. + set_source(Protobuf) +- resolve_dependency(Protobuf 3.21.4 EXACT) ++ resolve_dependency(Protobuf 3.21 EXACT) + include_directories(${Protobuf_INCLUDE_DIRS}) + endif() + diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index ce4c24dbe..785a2acc6 100644 --- a/third_party/CMakeLists.txt @@ -127,17 +136,6 @@ index 10ee508ba..027a58ecc 100644 setupEnvironment(hadoopHomeDirectory.string()); } -diff --git a/velox/dwio/common/CMakeLists.txt b/velox/dwio/common/CMakeLists.txt -index 9b6574d6e..61abddb59 100644 ---- a/velox/dwio/common/CMakeLists.txt -+++ b/velox/dwio/common/CMakeLists.txt -@@ -77,4 +77,5 @@ target_link_libraries( - velox_memory - Boost::regex - Folly::folly -- glog::glog) -+ glog::glog -+ protobuf::libprotobuf) diff --git a/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt b/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt index 2cabfc29a..54329ce23 100644 --- a/velox/dwio/parquet/writer/arrow/tests/CMakeLists.txt From 01d08bfa9d92fa47035f6d66f269d67a64aa4826 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Wed, 5 Jun 2024 18:23:21 +0800 Subject: [PATCH 213/402] [GLUTEN-3582][CH] Using ParquetBlockInputFormat instead of VectorizedParquetBlockInputFormat for complex type (#5995) [CH] Using ParquetBlockInputFormat instead of VectorizedParquetBlockInputFormat for complex type --- .../GlutenClickHouseHiveTableSuite.scala | 1 + .../SubstraitSource/ParquetFormatFile.cpp | 20 +++++++++++-- .../SubstraitSource/ParquetFormatFile.h | 4 ++- .../local-engine/tests/gtest_parquet_read.cpp | 29 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala index 3c993b622018..9b52f6a8cb53 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala @@ -111,6 +111,7 @@ class GlutenClickHouseHiveTableSuite getClass.getResource("/").getPath + "tests-working-home/spark-warehouse") .set("spark.hive.exec.dynamic.partition.mode", "nonstrict") .set("spark.gluten.supported.hive.udfs", "my_add") + .set("spark.gluten.sql.columnar.backend.ch.runtime_config.use_local_format", "true") .setMaster("local[*]") } diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp index 2e0f000456d8..f557df5b27bf 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include #include @@ -46,12 +46,13 @@ extern const int UNKNOWN_TYPE; namespace local_engine { + ParquetFormatFile::ParquetFormatFile( const DB::ContextPtr & context_, const substrait::ReadRel::LocalFiles::FileOrFiles & file_info_, const ReadBufferBuilderPtr & read_buffer_builder_, bool use_local_format_) - : FormatFile(context_, file_info_, read_buffer_builder_), use_local_format(use_local_format_) + : FormatFile(context_, file_info_, read_buffer_builder_), use_pageindex_reader(use_local_format_) { } @@ -85,7 +86,7 @@ FormatFile::InputFormatPtr ParquetFormatFile::createInputFormat(const DB::Block std::ranges::set_difference(total_row_group_indices, required_row_group_indices, std::back_inserter(skip_row_group_indices)); format_settings.parquet.skip_row_groups = std::unordered_set(skip_row_group_indices.begin(), skip_row_group_indices.end()); - if (use_local_format) + if (use_pageindex_reader && pageindex_reader_support(header)) res->input = std::make_shared(*(res->read_buffer), header, format_settings); else res->input = std::make_shared(*(res->read_buffer), header, format_settings, 1, 8192); @@ -112,6 +113,19 @@ std::optional ParquetFormatFile::getTotalRows() return total_rows; } } +bool ParquetFormatFile::pageindex_reader_support(const DB::Block & header) +{ + const auto result = std::ranges::find_if( + header, + [](DB::ColumnWithTypeAndName const & col) + { + const DB::DataTypePtr type_not_nullable = DB::removeNullable(col.type); + const DB::WhichDataType which(type_not_nullable); + return DB::isArray(which) || DB::isMap(which) || DB::isTuple(which); + }); + + return result == header.end(); +} std::vector ParquetFormatFile::collectRequiredRowGroups(int & total_row_groups) const { diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h index 045f0049d674..ba7f28883e65 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h +++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h @@ -55,8 +55,10 @@ class ParquetFormatFile : public FormatFile String getFileFormat() const override { return "Parquet"; } + static bool pageindex_reader_support(const DB::Block & header); + private: - bool use_local_format; + bool use_pageindex_reader; std::mutex mutex; std::optional total_rows; diff --git a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp index 94f28763e679..9623ffa98d28 100644 --- a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp +++ b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp @@ -15,6 +15,9 @@ * limitations under the License. */ +#include + + #include "config.h" #if USE_PARQUET @@ -139,6 +142,32 @@ TEST(ParquetRead, ReadSchema) readSchema("alltypes/alltypes_null.parquet"); } +TEST(ParquetRead, VerifyPageindexReaderSupport) +{ + EXPECT_FALSE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("alltypes/alltypes_notnull.parquet"))))); + EXPECT_FALSE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("alltypes/alltypes_null.parquet"))))); + + + EXPECT_FALSE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("array.parquet"))))); + EXPECT_TRUE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("date.parquet"))))); + EXPECT_TRUE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("datetime64.parquet"))))); + EXPECT_TRUE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("decimal.parquet"))))); + EXPECT_TRUE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("iris.parquet"))))); + EXPECT_FALSE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("map.parquet"))))); + EXPECT_TRUE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("sample.parquet"))))); + EXPECT_FALSE(local_engine::ParquetFormatFile::pageindex_reader_support( + toBlockRowType(local_engine::test::readParquetSchema(local_engine::test::data_file("struct.parquet"))))); +} + TEST(ParquetRead, ReadDataNotNull) { const std::map fields{ From 54aeb010d1da0bdc795838a5d4557d641e8298a5 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Wed, 5 Jun 2024 19:09:03 +0800 Subject: [PATCH 214/402] [CORE] Drop inputAdaptor in plan tree string (#5993) --- .../v1-bhj-ras/spark32/1.txt | 24 +-- .../v1-bhj-ras/spark32/10.txt | 62 +++--- .../v1-bhj-ras/spark32/11.txt | 56 +++-- .../v1-bhj-ras/spark32/12.txt | 39 ++-- .../v1-bhj-ras/spark32/13.txt | 52 +++-- .../v1-bhj-ras/spark32/14.txt | 28 ++- .../v1-bhj-ras/spark32/15.txt | 41 ++-- .../v1-bhj-ras/spark32/16.txt | 50 +++-- .../v1-bhj-ras/spark32/17.txt | 14 +- .../v1-bhj-ras/spark32/18.txt | 80 ++++---- .../v1-bhj-ras/spark32/19.txt | 28 ++- .../v1-bhj-ras/spark32/20.txt | 110 +++++----- .../v1-bhj-ras/spark32/21.txt | 90 ++++---- .../v1-bhj-ras/spark32/22.txt | 35 ++-- .../v1-bhj-ras/spark32/3.txt | 47 ++--- .../v1-bhj-ras/spark32/4.txt | 41 ++-- .../v1-bhj-ras/spark32/5.txt | 101 +++++---- .../v1-bhj-ras/spark32/6.txt | 13 +- .../v1-bhj-ras/spark32/7.txt | 93 ++++----- .../v1-bhj-ras/spark32/8.txt | 133 ++++++------ .../v1-bhj-ras/spark32/9.txt | 99 +++++---- .../v1-bhj-ras/spark33/1.txt | 24 +-- .../v1-bhj-ras/spark33/10.txt | 62 +++--- .../v1-bhj-ras/spark33/11.txt | 89 ++++---- .../v1-bhj-ras/spark33/12.txt | 39 ++-- .../v1-bhj-ras/spark33/13.txt | 52 +++-- .../v1-bhj-ras/spark33/14.txt | 28 ++- .../v1-bhj-ras/spark33/15.txt | 39 ++-- .../v1-bhj-ras/spark33/16.txt | 50 +++-- .../v1-bhj-ras/spark33/17.txt | 14 +- .../v1-bhj-ras/spark33/18.txt | 80 ++++---- .../v1-bhj-ras/spark33/19.txt | 28 ++- .../v1-bhj-ras/spark33/20.txt | 91 ++++---- .../v1-bhj-ras/spark33/21.txt | 90 ++++---- .../v1-bhj-ras/spark33/22.txt | 48 ++--- .../v1-bhj-ras/spark33/3.txt | 47 ++--- .../v1-bhj-ras/spark33/4.txt | 41 ++-- .../v1-bhj-ras/spark33/5.txt | 101 +++++---- .../v1-bhj-ras/spark33/6.txt | 13 +- .../v1-bhj-ras/spark33/7.txt | 93 ++++----- .../v1-bhj-ras/spark33/8.txt | 133 ++++++------ .../v1-bhj-ras/spark33/9.txt | 99 +++++---- .../v1-bhj-ras/spark34/1.txt | 24 +-- .../v1-bhj-ras/spark34/10.txt | 62 +++--- .../v1-bhj-ras/spark34/11.txt | 89 ++++---- .../v1-bhj-ras/spark34/12.txt | 39 ++-- .../v1-bhj-ras/spark34/13.txt | 52 +++-- .../v1-bhj-ras/spark34/14.txt | 28 ++- .../v1-bhj-ras/spark34/15.txt | 39 ++-- .../v1-bhj-ras/spark34/16.txt | 50 +++-- .../v1-bhj-ras/spark34/17.txt | 14 +- .../v1-bhj-ras/spark34/18.txt | 80 ++++---- .../v1-bhj-ras/spark34/19.txt | 28 ++- .../v1-bhj-ras/spark34/20.txt | 78 ++++--- .../v1-bhj-ras/spark34/21.txt | 90 ++++---- .../v1-bhj-ras/spark34/22.txt | 48 ++--- .../v1-bhj-ras/spark34/3.txt | 47 ++--- .../v1-bhj-ras/spark34/4.txt | 41 ++-- .../v1-bhj-ras/spark34/5.txt | 101 +++++---- .../v1-bhj-ras/spark34/6.txt | 13 +- .../v1-bhj-ras/spark34/7.txt | 93 ++++----- .../v1-bhj-ras/spark34/8.txt | 133 ++++++------ .../v1-bhj-ras/spark34/9.txt | 99 +++++---- .../tpch-approved-plan/v1-bhj/spark32/1.txt | 24 +-- .../tpch-approved-plan/v1-bhj/spark32/10.txt | 62 +++--- .../tpch-approved-plan/v1-bhj/spark32/11.txt | 56 +++-- .../tpch-approved-plan/v1-bhj/spark32/12.txt | 39 ++-- .../tpch-approved-plan/v1-bhj/spark32/13.txt | 52 +++-- .../tpch-approved-plan/v1-bhj/spark32/14.txt | 28 ++- .../tpch-approved-plan/v1-bhj/spark32/15.txt | 41 ++-- .../tpch-approved-plan/v1-bhj/spark32/16.txt | 50 +++-- .../tpch-approved-plan/v1-bhj/spark32/17.txt | 14 +- .../tpch-approved-plan/v1-bhj/spark32/18.txt | 80 ++++---- .../tpch-approved-plan/v1-bhj/spark32/19.txt | 28 ++- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 110 +++++----- .../tpch-approved-plan/v1-bhj/spark32/21.txt | 90 ++++---- .../tpch-approved-plan/v1-bhj/spark32/22.txt | 35 ++-- .../tpch-approved-plan/v1-bhj/spark32/3.txt | 47 ++--- .../tpch-approved-plan/v1-bhj/spark32/4.txt | 41 ++-- .../tpch-approved-plan/v1-bhj/spark32/5.txt | 101 +++++---- .../tpch-approved-plan/v1-bhj/spark32/6.txt | 13 +- .../tpch-approved-plan/v1-bhj/spark32/7.txt | 93 ++++----- .../tpch-approved-plan/v1-bhj/spark32/8.txt | 133 ++++++------ .../tpch-approved-plan/v1-bhj/spark32/9.txt | 99 +++++---- .../tpch-approved-plan/v1-bhj/spark33/1.txt | 24 +-- .../tpch-approved-plan/v1-bhj/spark33/10.txt | 62 +++--- .../tpch-approved-plan/v1-bhj/spark33/11.txt | 89 ++++---- .../tpch-approved-plan/v1-bhj/spark33/12.txt | 39 ++-- .../tpch-approved-plan/v1-bhj/spark33/13.txt | 52 +++-- .../tpch-approved-plan/v1-bhj/spark33/14.txt | 28 ++- .../tpch-approved-plan/v1-bhj/spark33/15.txt | 39 ++-- .../tpch-approved-plan/v1-bhj/spark33/16.txt | 50 +++-- .../tpch-approved-plan/v1-bhj/spark33/17.txt | 14 +- .../tpch-approved-plan/v1-bhj/spark33/18.txt | 80 ++++---- .../tpch-approved-plan/v1-bhj/spark33/19.txt | 28 ++- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 91 ++++---- .../tpch-approved-plan/v1-bhj/spark33/21.txt | 90 ++++---- .../tpch-approved-plan/v1-bhj/spark33/22.txt | 61 +++--- .../tpch-approved-plan/v1-bhj/spark33/3.txt | 47 ++--- .../tpch-approved-plan/v1-bhj/spark33/4.txt | 41 ++-- .../tpch-approved-plan/v1-bhj/spark33/5.txt | 101 +++++---- .../tpch-approved-plan/v1-bhj/spark33/6.txt | 13 +- .../tpch-approved-plan/v1-bhj/spark33/7.txt | 93 ++++----- .../tpch-approved-plan/v1-bhj/spark33/8.txt | 133 ++++++------ .../tpch-approved-plan/v1-bhj/spark33/9.txt | 99 +++++---- .../tpch-approved-plan/v1-bhj/spark34/1.txt | 24 +-- .../tpch-approved-plan/v1-bhj/spark34/10.txt | 62 +++--- .../tpch-approved-plan/v1-bhj/spark34/11.txt | 89 ++++---- .../tpch-approved-plan/v1-bhj/spark34/12.txt | 39 ++-- .../tpch-approved-plan/v1-bhj/spark34/13.txt | 52 +++-- .../tpch-approved-plan/v1-bhj/spark34/14.txt | 28 ++- .../tpch-approved-plan/v1-bhj/spark34/15.txt | 39 ++-- .../tpch-approved-plan/v1-bhj/spark34/16.txt | 50 +++-- .../tpch-approved-plan/v1-bhj/spark34/17.txt | 14 +- .../tpch-approved-plan/v1-bhj/spark34/18.txt | 80 ++++---- .../tpch-approved-plan/v1-bhj/spark34/19.txt | 28 ++- .../tpch-approved-plan/v1-bhj/spark34/20.txt | 78 ++++--- .../tpch-approved-plan/v1-bhj/spark34/21.txt | 90 ++++---- .../tpch-approved-plan/v1-bhj/spark34/22.txt | 61 +++--- .../tpch-approved-plan/v1-bhj/spark34/3.txt | 47 ++--- .../tpch-approved-plan/v1-bhj/spark34/4.txt | 41 ++-- .../tpch-approved-plan/v1-bhj/spark34/5.txt | 101 +++++---- .../tpch-approved-plan/v1-bhj/spark34/6.txt | 13 +- .../tpch-approved-plan/v1-bhj/spark34/7.txt | 93 ++++----- .../tpch-approved-plan/v1-bhj/spark34/8.txt | 133 ++++++------ .../tpch-approved-plan/v1-bhj/spark34/9.txt | 99 +++++---- .../tpch-approved-plan/v1-ras/spark32/1.txt | 24 +-- .../tpch-approved-plan/v1-ras/spark32/10.txt | 87 ++++---- .../tpch-approved-plan/v1-ras/spark32/11.txt | 74 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 48 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 52 +++-- .../tpch-approved-plan/v1-ras/spark32/14.txt | 22 +- .../tpch-approved-plan/v1-ras/spark32/15.txt | 43 ++-- .../tpch-approved-plan/v1-ras/spark32/16.txt | 61 +++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 35 ++-- .../tpch-approved-plan/v1-ras/spark32/18.txt | 81 ++++---- .../tpch-approved-plan/v1-ras/spark32/19.txt | 22 +- .../tpch-approved-plan/v1-ras/spark32/20.txt | 133 ++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 124 +++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 46 ++--- .../tpch-approved-plan/v1-ras/spark32/3.txt | 46 ++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 48 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 144 ++++++------- .../tpch-approved-plan/v1-ras/spark32/6.txt | 13 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 138 ++++++------- .../tpch-approved-plan/v1-ras/spark32/8.txt | 194 ++++++++---------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 144 ++++++------- .../tpch-approved-plan/v1-ras/spark33/1.txt | 24 +-- .../tpch-approved-plan/v1-ras/spark33/10.txt | 87 ++++---- .../tpch-approved-plan/v1-ras/spark33/11.txt | 108 +++++----- .../tpch-approved-plan/v1-ras/spark33/12.txt | 48 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 52 +++-- .../tpch-approved-plan/v1-ras/spark33/14.txt | 22 +- .../tpch-approved-plan/v1-ras/spark33/15.txt | 41 ++-- .../tpch-approved-plan/v1-ras/spark33/16.txt | 61 +++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 35 ++-- .../tpch-approved-plan/v1-ras/spark33/18.txt | 81 ++++---- .../tpch-approved-plan/v1-ras/spark33/19.txt | 22 +- .../tpch-approved-plan/v1-ras/spark33/20.txt | 120 +++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 124 +++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 59 +++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 46 ++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 48 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 144 ++++++------- .../tpch-approved-plan/v1-ras/spark33/6.txt | 13 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 138 ++++++------- .../tpch-approved-plan/v1-ras/spark33/8.txt | 194 ++++++++---------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 144 ++++++------- .../tpch-approved-plan/v1-ras/spark34/1.txt | 24 +-- .../tpch-approved-plan/v1-ras/spark34/10.txt | 87 ++++---- .../tpch-approved-plan/v1-ras/spark34/11.txt | 108 +++++----- .../tpch-approved-plan/v1-ras/spark34/12.txt | 48 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 52 +++-- .../tpch-approved-plan/v1-ras/spark34/14.txt | 22 +- .../tpch-approved-plan/v1-ras/spark34/15.txt | 41 ++-- .../tpch-approved-plan/v1-ras/spark34/16.txt | 61 +++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 35 ++-- .../tpch-approved-plan/v1-ras/spark34/18.txt | 81 ++++---- .../tpch-approved-plan/v1-ras/spark34/19.txt | 22 +- .../tpch-approved-plan/v1-ras/spark34/20.txt | 120 +++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 124 +++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 59 +++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 46 ++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 48 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 144 ++++++------- .../tpch-approved-plan/v1-ras/spark34/6.txt | 13 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 138 ++++++------- .../tpch-approved-plan/v1-ras/spark34/8.txt | 194 ++++++++---------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 144 ++++++------- .../tpch-approved-plan/v1/spark32/1.txt | 24 +-- .../tpch-approved-plan/v1/spark32/10.txt | 87 ++++---- .../tpch-approved-plan/v1/spark32/11.txt | 74 +++---- .../tpch-approved-plan/v1/spark32/12.txt | 48 ++--- .../tpch-approved-plan/v1/spark32/13.txt | 52 +++-- .../tpch-approved-plan/v1/spark32/14.txt | 22 +- .../tpch-approved-plan/v1/spark32/15.txt | 43 ++-- .../tpch-approved-plan/v1/spark32/16.txt | 61 +++--- .../tpch-approved-plan/v1/spark32/17.txt | 35 ++-- .../tpch-approved-plan/v1/spark32/18.txt | 81 ++++---- .../tpch-approved-plan/v1/spark32/19.txt | 22 +- .../tpch-approved-plan/v1/spark32/20.txt | 133 ++++++------ .../tpch-approved-plan/v1/spark32/21.txt | 124 +++++------ .../tpch-approved-plan/v1/spark32/22.txt | 46 ++--- .../tpch-approved-plan/v1/spark32/3.txt | 46 ++--- .../tpch-approved-plan/v1/spark32/4.txt | 48 ++--- .../tpch-approved-plan/v1/spark32/5.txt | 144 ++++++------- .../tpch-approved-plan/v1/spark32/6.txt | 13 +- .../tpch-approved-plan/v1/spark32/7.txt | 138 ++++++------- .../tpch-approved-plan/v1/spark32/8.txt | 194 ++++++++---------- .../tpch-approved-plan/v1/spark32/9.txt | 144 ++++++------- .../tpch-approved-plan/v1/spark33/1.txt | 24 +-- .../tpch-approved-plan/v1/spark33/10.txt | 87 ++++---- .../tpch-approved-plan/v1/spark33/11.txt | 108 +++++----- .../tpch-approved-plan/v1/spark33/12.txt | 48 ++--- .../tpch-approved-plan/v1/spark33/13.txt | 52 +++-- .../tpch-approved-plan/v1/spark33/14.txt | 22 +- .../tpch-approved-plan/v1/spark33/15.txt | 41 ++-- .../tpch-approved-plan/v1/spark33/16.txt | 61 +++--- .../tpch-approved-plan/v1/spark33/17.txt | 35 ++-- .../tpch-approved-plan/v1/spark33/18.txt | 81 ++++---- .../tpch-approved-plan/v1/spark33/19.txt | 22 +- .../tpch-approved-plan/v1/spark33/20.txt | 120 +++++------ .../tpch-approved-plan/v1/spark33/21.txt | 124 +++++------ .../tpch-approved-plan/v1/spark33/22.txt | 72 +++---- .../tpch-approved-plan/v1/spark33/3.txt | 46 ++--- .../tpch-approved-plan/v1/spark33/4.txt | 48 ++--- .../tpch-approved-plan/v1/spark33/5.txt | 144 ++++++------- .../tpch-approved-plan/v1/spark33/6.txt | 13 +- .../tpch-approved-plan/v1/spark33/7.txt | 138 ++++++------- .../tpch-approved-plan/v1/spark33/8.txt | 194 ++++++++---------- .../tpch-approved-plan/v1/spark33/9.txt | 144 ++++++------- .../tpch-approved-plan/v1/spark34/1.txt | 24 +-- .../tpch-approved-plan/v1/spark34/10.txt | 87 ++++---- .../tpch-approved-plan/v1/spark34/11.txt | 108 +++++----- .../tpch-approved-plan/v1/spark34/12.txt | 48 ++--- .../tpch-approved-plan/v1/spark34/13.txt | 52 +++-- .../tpch-approved-plan/v1/spark34/14.txt | 22 +- .../tpch-approved-plan/v1/spark34/15.txt | 41 ++-- .../tpch-approved-plan/v1/spark34/16.txt | 61 +++--- .../tpch-approved-plan/v1/spark34/17.txt | 35 ++-- .../tpch-approved-plan/v1/spark34/18.txt | 81 ++++---- .../tpch-approved-plan/v1/spark34/19.txt | 22 +- .../tpch-approved-plan/v1/spark34/20.txt | 120 +++++------ .../tpch-approved-plan/v1/spark34/21.txt | 124 +++++------ .../tpch-approved-plan/v1/spark34/22.txt | 72 +++---- .../tpch-approved-plan/v1/spark34/3.txt | 46 ++--- .../tpch-approved-plan/v1/spark34/4.txt | 48 ++--- .../tpch-approved-plan/v1/spark34/5.txt | 144 ++++++------- .../tpch-approved-plan/v1/spark34/6.txt | 13 +- .../tpch-approved-plan/v1/spark34/7.txt | 138 ++++++------- .../tpch-approved-plan/v1/spark34/8.txt | 194 ++++++++---------- .../tpch-approved-plan/v1/spark34/9.txt | 144 ++++++------- .../execution/WholeStageTransformer.scala | 2 +- .../ColumnarCollapseTransformStages.scala | 2 +- .../execution/GenerateTreeStringShim.scala | 29 ++- .../execution/GenerateTreeStringShim.scala | 29 ++- .../execution/GenerateTreeStringShim.scala | 29 ++- .../execution/GenerateTreeStringShim.scala | 29 ++- 258 files changed, 8004 insertions(+), 9260 deletions(-) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 8d67aad16c3d..9316f6f8ebb5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index 512aa92207cd..d67e373daf9b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -6,39 +6,35 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (40) +- ^ RegularHashAggregateExecTransformer (39) +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + +- ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == TakeOrderedAndProject (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index a330846ccf90..f65747c37e65 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -4,36 +4,32 @@ AdaptiveSparkPlan (58) VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == Sort (57) +- Exchange (56) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index dea6e04c33ff..f1b7bbfe6acb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -4,27 +4,24 @@ AdaptiveSparkPlan (42) VeloxColumnarToRowExec (28) +- ^ SortExecTransformer (26) +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (41) +- Exchange (40) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 82786dc4457b..12147d4b3197 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index 7de47d57f6fd..87dc45754b66 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -5,21 +5,19 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (33) +- Exchange (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index c16f304be090..0bfcd38f02d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index 08eb4f850f22..ccc5488215a6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -4,33 +4,29 @@ AdaptiveSparkPlan (56) VeloxColumnarToRowExec (35) +- ^ SortExecTransformer (33) +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (55) +- Exchange (54) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt index d806c8c2577b..9d81e1d5053f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt @@ -5,14 +5,12 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (12) +- ^ RegularHashAggregateExecTransformer (11) +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- ^ InputAdapter (3) - +- ^ RowToVeloxColumnar (2) - +- ^ LocalTableScan (1) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == HashAggregate (34) +- Exchange (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index 8dd094081ad3..e582fd47cf55 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -5,49 +5,43 @@ AdaptiveSparkPlan (86) +- TakeOrderedAndProjectExecTransformer (52) +- ^ RegularHashAggregateExecTransformer (50) +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32) - +- ReusedExchange (31) + +- ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ShuffleQueryStage (15) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- BroadcastQueryStage (32) + +- ReusedExchange (31) +- == Initial Plan == TakeOrderedAndProject (85) +- HashAggregate (84) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 289def220b08..72a9f74f86ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -4,21 +4,19 @@ AdaptiveSparkPlan (33) VeloxColumnarToRowExec (21) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (32) +- Exchange (31) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index 71889beea972..e09911ea6ad8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -4,65 +4,57 @@ AdaptiveSparkPlan (107) VeloxColumnarToRowExec (70) +- ^ SortExecTransformer (68) +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ AQEShuffleRead (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- ^ InputAdapter (18) - : : +- ^ BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ^ InputAdapter (39) - : +- ^ ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- ^ InputAdapter (59) - +- ^ BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + +- ShuffleQueryStage (65) + +- ColumnarExchange (64) + +- ^ ProjectExecTransformer (62) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) + :- ^ ProjectExecTransformer (52) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) + : :- ^ InputIteratorTransformer (9) + : : +- AQEShuffleRead (7) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (50) + : +- BroadcastQueryStage (48) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) + : :- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) + : : :- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- BroadcastQueryStage (17) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ NoopFilter (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ShuffleQueryStage (38) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ NoopFilter (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- BroadcastQueryStage (30) + : +- ReusedExchange (29) + +- ^ InputIteratorTransformer (60) + +- BroadcastQueryStage (58) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == Sort (106) +- Exchange (105) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index bd977b45da55..10e08cfd60b5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -5,54 +5,48 @@ AdaptiveSparkPlan (92) +- TakeOrderedAndProjectExecTransformer (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (91) +- HashAggregate (90) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index 4b7d882ca23e..a8c66bef1716 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -4,25 +4,22 @@ AdaptiveSparkPlan (38) VeloxColumnarToRowExec (26) +- ^ SortExecTransformer (24) +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (37) +- Exchange (36) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index e5f266cc71f3..b9b7951d7652 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -6,31 +6,28 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (31) +- ^ RegularHashAggregateExecTransformer (30) +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == TakeOrderedAndProject (52) +- HashAggregate (51) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 2bb4d8907de7..97d18724dc9d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index 6ce90735fbeb..a9b993d0a078 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -4,60 +4,53 @@ AdaptiveSparkPlan (100) VeloxColumnarToRowExec (65) +- ^ SortExecTransformer (63) +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (99) +- Exchange (98) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index 8d1a71e9751e..7aae3dccfd9b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index ae4b171974b1..b0d68672ca3a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -4,56 +4,49 @@ AdaptiveSparkPlan (93) VeloxColumnarToRowExec (60) +- ^ SortExecTransformer (58) +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40) - +- ReusedExchange (39) + +- ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == Sort (92) +- Exchange (91) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index ce470033eb1d..7e20b94edaa7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -4,77 +4,68 @@ AdaptiveSparkPlan (129) VeloxColumnarToRowExec (84) +- ^ SortExecTransformer (82) +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + +- ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == Sort (128) +- Exchange (127) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index 3744b8895539..8a30fbdfb152 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -4,59 +4,52 @@ AdaptiveSparkPlan (98) VeloxColumnarToRowExec (64) +- ^ SortExecTransformer (62) +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (97) +- Exchange (96) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index 74f79bd3ee64..63b30bb5d26b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 0c1832920c86..18c2ef9bc473 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -6,39 +6,35 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (40) +- ^ RegularHashAggregateExecTransformer (39) +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + +- ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == TakeOrderedAndProject (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index b66d0bb6a930..6afbee587ed2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -4,36 +4,32 @@ AdaptiveSparkPlan (58) VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == Sort (57) +- Exchange (56) @@ -322,24 +318,21 @@ AdaptiveSparkPlan (99) +- ^ ProjectExecTransformer (80) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) + : :- ^ NoopFilter (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == HashAggregate (98) +- Exchange (97) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index 073b9de3885e..c65916d66898 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -4,27 +4,24 @@ AdaptiveSparkPlan (42) VeloxColumnarToRowExec (28) +- ^ SortExecTransformer (26) +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (41) +- Exchange (40) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index baa569ccebe9..9ff054f9314e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index 699ef76d14e6..cc14a0347f1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -5,21 +5,19 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (33) +- Exchange (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index fdb22283555b..2d463a561ab6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -8,22 +8,20 @@ AdaptiveSparkPlan (41) +- ^ ProjectExecTransformer (21) +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (40) +- Exchange (39) @@ -237,14 +235,13 @@ AdaptiveSparkPlan (66) +- ^ ProjectExecTransformer (53) +- ^ RegularHashAggregateExecTransformer (52) +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ NoopFilter (43) + +- ^ Scan parquet (42) +- == Initial Plan == HashAggregate (65) +- HashAggregate (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index 05b0b2f9723a..c4d21c42e70b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -4,33 +4,29 @@ AdaptiveSparkPlan (56) VeloxColumnarToRowExec (35) +- ^ SortExecTransformer (33) +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (55) +- Exchange (54) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt index 712560e7f577..0b41400d5864 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt @@ -5,14 +5,12 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (12) +- ^ RegularHashAggregateExecTransformer (11) +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- ^ InputAdapter (3) - +- ^ RowToVeloxColumnar (2) - +- ^ LocalTableScan (1) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == HashAggregate (34) +- Exchange (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index c21e377b0153..8f03bd54cd2d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -5,49 +5,43 @@ AdaptiveSparkPlan (86) +- TakeOrderedAndProjectExecTransformer (52) +- ^ RegularHashAggregateExecTransformer (50) +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + +- ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == TakeOrderedAndProject (85) +- HashAggregate (84) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index bce0e46e72dc..1fcd294716ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -4,21 +4,19 @@ AdaptiveSparkPlan (33) VeloxColumnarToRowExec (21) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (32) +- Exchange (31) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index d4aec5f07beb..ee30d646767e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -10,57 +10,50 @@ AdaptiveSparkPlan (104) :- ^ ProjectExecTransformer (52) : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) : :- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ AQEShuffleRead (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) + : : +- AQEShuffleRead (7) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- ^ InputAdapter (18) - : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ^ InputAdapter (39) - : +- ^ ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) + : +- BroadcastQueryStage (48), Statistics(X) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) + : :- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) + : : :- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- BroadcastQueryStage (17), Statistics(X) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ NoopFilter (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ShuffleQueryStage (38), Statistics(X) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ NoopFilter (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- BroadcastQueryStage (30), Statistics(X) + : +- ReusedExchange (29) +- ^ InputIteratorTransformer (60) - +- ^ InputAdapter (59) - +- ^ BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + +- BroadcastQueryStage (58), Statistics(X) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == Sort (103) +- Exchange (102) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index 361860c7486b..c52c51139d2a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -4,54 +4,48 @@ AdaptiveSparkPlan (91) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (90) +- HashAggregate (89) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index c0a11106bbf9..d404e2149a89 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -4,25 +4,22 @@ AdaptiveSparkPlan (38) VeloxColumnarToRowExec (26) +- ^ SortExecTransformer (24) +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (37) +- Exchange (36) @@ -214,13 +211,12 @@ AdaptiveSparkPlan (57) VeloxColumnarToRowExec (50) +- ^ RegularHashAggregateExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == HashAggregate (56) +- Exchange (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index 153e81420430..dbe67ba34d04 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -6,31 +6,28 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (31) +- ^ RegularHashAggregateExecTransformer (30) +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == TakeOrderedAndProject (52) +- HashAggregate (51) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index ccb61f1c6cd3..626f608cecd0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index b12ece606bc2..7a2ce7cf7a4a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -4,60 +4,53 @@ AdaptiveSparkPlan (100) VeloxColumnarToRowExec (65) +- ^ SortExecTransformer (63) +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (99) +- Exchange (98) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index 51c5836bdd11..e03830992c2e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 3fec6f0c8b2c..4f9b5f20956d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -4,56 +4,49 @@ AdaptiveSparkPlan (93) VeloxColumnarToRowExec (60) +- ^ SortExecTransformer (58) +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == Sort (92) +- Exchange (91) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index da6e7a768f23..677a42eb711a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -4,77 +4,68 @@ AdaptiveSparkPlan (129) VeloxColumnarToRowExec (84) +- ^ SortExecTransformer (82) +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == Sort (128) +- Exchange (127) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 6e93f3a79f22..9f2111b015ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -4,59 +4,52 @@ AdaptiveSparkPlan (98) VeloxColumnarToRowExec (64) +- ^ SortExecTransformer (62) +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (97) +- Exchange (96) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index 41613c36f7bd..c254ec8c82ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index b917f359d1a7..ffc37f5b7908 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -6,39 +6,35 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (40) +- ^ RegularHashAggregateExecTransformer (39) +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + +- ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == TakeOrderedAndProject (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index 50c599a3b051..6410bb3fed5f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -4,36 +4,32 @@ AdaptiveSparkPlan (58) VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == Sort (57) +- Exchange (56) @@ -326,24 +322,21 @@ AdaptiveSparkPlan (99) +- ^ ProjectExecTransformer (80) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) + : :- ^ NoopFilter (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == HashAggregate (98) +- Exchange (97) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index c57f98c1ff85..4e85d8194098 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -4,27 +4,24 @@ AdaptiveSparkPlan (42) VeloxColumnarToRowExec (28) +- ^ SortExecTransformer (26) +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (41) +- Exchange (40) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index b29e27ae092b..e83ab1aca83c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index 4b3e5634106f..58d532222af5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -5,21 +5,19 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (33) +- Exchange (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 27eec9ac6340..269921602469 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -8,22 +8,20 @@ AdaptiveSparkPlan (41) +- ^ ProjectExecTransformer (21) +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (40) +- Exchange (39) @@ -239,14 +237,13 @@ AdaptiveSparkPlan (66) +- ^ ProjectExecTransformer (53) +- ^ RegularHashAggregateExecTransformer (52) +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ NoopFilter (43) + +- ^ Scan parquet (42) +- == Initial Plan == HashAggregate (65) +- HashAggregate (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index fdec20383624..0a8771619020 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -4,33 +4,29 @@ AdaptiveSparkPlan (56) VeloxColumnarToRowExec (35) +- ^ SortExecTransformer (33) +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (55) +- Exchange (54) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt index 4a2b5a1744e0..62612096db42 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt @@ -5,14 +5,12 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (12) +- ^ RegularHashAggregateExecTransformer (11) +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- ^ InputAdapter (3) - +- ^ RowToVeloxColumnar (2) - +- ^ LocalTableScan (1) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == HashAggregate (34) +- Exchange (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index 66819677c834..f0420b98194d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -5,49 +5,43 @@ AdaptiveSparkPlan (86) +- TakeOrderedAndProjectExecTransformer (52) +- ^ RegularHashAggregateExecTransformer (50) +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + +- ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == TakeOrderedAndProject (85) +- HashAggregate (84) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index 9b8041d6e824..c39011756bca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -4,21 +4,19 @@ AdaptiveSparkPlan (33) VeloxColumnarToRowExec (21) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (32) +- Exchange (31) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index e1724af1c30c..6bad140301d8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -12,49 +12,43 @@ AdaptiveSparkPlan (96) : :- ^ NoopFilter (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- ^ InputAdapter (17) - : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ NoopFilter (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- ^ InputAdapter (11) - : : +- ^ BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ NoopFilter (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ^ InputAdapter (32) - : +- ^ ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) + : +- BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ NoopFilter (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ NoopFilter (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + +- BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == Sort (95) +- Exchange (94) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index c418b8c4fba4..14b23aa966e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -4,54 +4,48 @@ AdaptiveSparkPlan (91) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (90) +- HashAggregate (89) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 953ec6a34a55..4517787bcaef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -4,25 +4,22 @@ AdaptiveSparkPlan (38) VeloxColumnarToRowExec (26) +- ^ SortExecTransformer (24) +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (37) +- Exchange (36) @@ -216,13 +213,12 @@ AdaptiveSparkPlan (57) VeloxColumnarToRowExec (50) +- ^ RegularHashAggregateExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == HashAggregate (56) +- Exchange (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index af59560de7d4..361ea070d42a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -6,31 +6,28 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (31) +- ^ RegularHashAggregateExecTransformer (30) +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == TakeOrderedAndProject (52) +- HashAggregate (51) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 1e7d5f6793cf..4bdcd640058c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index 170e4d012be3..d1f0cb5eba56 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -4,60 +4,53 @@ AdaptiveSparkPlan (100) VeloxColumnarToRowExec (65) +- ^ SortExecTransformer (63) +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (99) +- Exchange (98) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 9333e2a8ad59..5987a808f5fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 9b6036312f19..29c2524a7615 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -4,56 +4,49 @@ AdaptiveSparkPlan (93) VeloxColumnarToRowExec (60) +- ^ SortExecTransformer (58) +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == Sort (92) +- Exchange (91) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index 48924df3085b..715bc6f4e5f8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -4,77 +4,68 @@ AdaptiveSparkPlan (129) VeloxColumnarToRowExec (84) +- ^ SortExecTransformer (82) +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == Sort (128) +- Exchange (127) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 7e7b4390eefe..14c32b04eb48 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -4,59 +4,52 @@ AdaptiveSparkPlan (98) VeloxColumnarToRowExec (64) +- ^ SortExecTransformer (62) +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (97) +- Exchange (96) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt index 7a80ed95f3e8..656f98574483 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index c6d18d83e37a..3afadbc4c678 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -6,39 +6,35 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (40) +- ^ RegularHashAggregateExecTransformer (39) +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + +- ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == TakeOrderedAndProject (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt index c147c286d9cf..03199cd9feb1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt @@ -4,36 +4,32 @@ AdaptiveSparkPlan (58) VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == Sort (57) +- Exchange (56) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt index cdd0b5527b0c..e78891a522ff 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt @@ -4,27 +4,24 @@ AdaptiveSparkPlan (42) VeloxColumnarToRowExec (28) +- ^ SortExecTransformer (26) +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (41) +- Exchange (40) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt index a0733090396d..9aa658b2d78d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt index 567e2ba8fa8f..fcd550bdb92c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt @@ -5,21 +5,19 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (33) +- Exchange (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt index 1d63d95f9c29..54dad76174f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt index f9c811ab7bd1..5197f57218a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt @@ -4,33 +4,29 @@ AdaptiveSparkPlan (56) VeloxColumnarToRowExec (35) +- ^ SortExecTransformer (33) +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (55) +- Exchange (54) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt index d806c8c2577b..9d81e1d5053f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt @@ -5,14 +5,12 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (12) +- ^ RegularHashAggregateExecTransformer (11) +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- ^ InputAdapter (3) - +- ^ RowToVeloxColumnar (2) - +- ^ LocalTableScan (1) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == HashAggregate (34) +- Exchange (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index 53d2e12db19b..341589565740 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -5,49 +5,43 @@ AdaptiveSparkPlan (86) +- TakeOrderedAndProjectExecTransformer (52) +- ^ RegularHashAggregateExecTransformer (50) +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32) - +- ReusedExchange (31) + +- ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ShuffleQueryStage (15) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) + :- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- BroadcastQueryStage (32) + +- ReusedExchange (31) +- == Initial Plan == TakeOrderedAndProject (85) +- HashAggregate (84) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt index 15cc941c36de..a46daacbdb75 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt @@ -4,21 +4,19 @@ AdaptiveSparkPlan (33) VeloxColumnarToRowExec (21) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (32) +- Exchange (31) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index 8af0e8b1d511..6fcedbbda996 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -4,65 +4,57 @@ AdaptiveSparkPlan (107) VeloxColumnarToRowExec (70) +- ^ SortExecTransformer (68) +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ AQEShuffleRead (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- ^ InputAdapter (18) - : : +- ^ BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ^ InputAdapter (39) - : +- ^ ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- ^ InputAdapter (59) - +- ^ BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + +- ShuffleQueryStage (65) + +- ColumnarExchange (64) + +- ^ ProjectExecTransformer (62) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) + :- ^ ProjectExecTransformer (52) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) + : :- ^ InputIteratorTransformer (9) + : : +- AQEShuffleRead (7) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (50) + : +- BroadcastQueryStage (48) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) + : :- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) + : : :- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- BroadcastQueryStage (17) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ FilterExecTransformer (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ShuffleQueryStage (38) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ FilterExecTransformer (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- BroadcastQueryStage (30) + : +- ReusedExchange (29) + +- ^ InputIteratorTransformer (60) + +- BroadcastQueryStage (58) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ Scan parquet (53) +- == Initial Plan == Sort (106) +- Exchange (105) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index bf33da8261c8..4a970927299d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -5,54 +5,48 @@ AdaptiveSparkPlan (92) +- TakeOrderedAndProjectExecTransformer (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (91) +- HashAggregate (90) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt index 3723f8f01625..d0252d7ac997 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt @@ -4,25 +4,22 @@ AdaptiveSparkPlan (38) VeloxColumnarToRowExec (26) +- ^ SortExecTransformer (24) +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (37) +- Exchange (36) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index 373882a24e57..ba63341e7e03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -6,31 +6,28 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (31) +- ^ RegularHashAggregateExecTransformer (30) +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == TakeOrderedAndProject (52) +- HashAggregate (51) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt index adf7259779a5..54c5c1b24d8b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt index cfed4af71090..2f037bd9c1d1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt @@ -4,60 +4,53 @@ AdaptiveSparkPlan (100) VeloxColumnarToRowExec (65) +- ^ SortExecTransformer (63) +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (99) +- Exchange (98) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt index fa2536a365c8..b39d8c1b2aec 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt index 910190a65e0f..4b2aa1e644a2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt @@ -4,56 +4,49 @@ AdaptiveSparkPlan (93) VeloxColumnarToRowExec (60) +- ^ SortExecTransformer (58) +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40) - +- ReusedExchange (39) + +- ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == Sort (92) +- Exchange (91) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt index f16be6bdb2e5..fe9d78579598 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt @@ -4,77 +4,68 @@ AdaptiveSparkPlan (129) VeloxColumnarToRowExec (84) +- ^ SortExecTransformer (82) +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + +- ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == Sort (128) +- Exchange (127) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt index 22233dbc5ae2..cb62cd97d3e0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt @@ -4,59 +4,52 @@ AdaptiveSparkPlan (98) VeloxColumnarToRowExec (64) +- ^ SortExecTransformer (62) +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (97) +- Exchange (96) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt index 0cacd70fdca7..89de3133895b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index ebb947ebb002..898458b34cb7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -6,39 +6,35 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (40) +- ^ RegularHashAggregateExecTransformer (39) +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + +- ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == TakeOrderedAndProject (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt index 6ab05c3c2261..6677083c130d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt @@ -4,36 +4,32 @@ AdaptiveSparkPlan (58) VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == Sort (57) +- Exchange (56) @@ -322,24 +318,21 @@ AdaptiveSparkPlan (99) +- ^ ProjectExecTransformer (80) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) + : :- ^ FilterExecTransformer (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == HashAggregate (98) +- Exchange (97) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt index 98272b12cebc..e83423c0b2f2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt @@ -4,27 +4,24 @@ AdaptiveSparkPlan (42) VeloxColumnarToRowExec (28) +- ^ SortExecTransformer (26) +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (41) +- Exchange (40) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt index c1919f2e620d..1fa2d94b096b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt index 17283b824590..5bc1aef67790 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt @@ -5,21 +5,19 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (33) +- Exchange (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt index ce87f94218ef..5cd45806c9fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt @@ -8,22 +8,20 @@ AdaptiveSparkPlan (41) +- ^ ProjectExecTransformer (21) +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (40) +- Exchange (39) @@ -237,14 +235,13 @@ AdaptiveSparkPlan (66) +- ^ ProjectExecTransformer (53) +- ^ RegularHashAggregateExecTransformer (52) +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ FilterExecTransformer (43) + +- ^ Scan parquet (42) +- == Initial Plan == HashAggregate (65) +- HashAggregate (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt index 6a460d28f171..56d39742c76e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt @@ -4,33 +4,29 @@ AdaptiveSparkPlan (56) VeloxColumnarToRowExec (35) +- ^ SortExecTransformer (33) +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (55) +- Exchange (54) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt index 712560e7f577..0b41400d5864 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt @@ -5,14 +5,12 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (12) +- ^ RegularHashAggregateExecTransformer (11) +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- ^ InputAdapter (3) - +- ^ RowToVeloxColumnar (2) - +- ^ LocalTableScan (1) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == HashAggregate (34) +- Exchange (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index 884bba49cbf3..c37f0b49567c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -5,49 +5,43 @@ AdaptiveSparkPlan (86) +- TakeOrderedAndProjectExecTransformer (52) +- ^ RegularHashAggregateExecTransformer (50) +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + +- ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) + :- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == TakeOrderedAndProject (85) +- HashAggregate (84) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt index b78310babb5b..6822887d3aed 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt @@ -4,21 +4,19 @@ AdaptiveSparkPlan (33) VeloxColumnarToRowExec (21) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (32) +- Exchange (31) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index f71a534d99d2..32f28a0311c4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -10,57 +10,50 @@ AdaptiveSparkPlan (104) :- ^ ProjectExecTransformer (52) : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) : :- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ AQEShuffleRead (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) + : : +- AQEShuffleRead (7) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- ^ InputAdapter (18) - : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ^ InputAdapter (39) - : +- ^ ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) + : +- BroadcastQueryStage (48), Statistics(X) + : +- ColumnarBroadcastExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) + : :- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) + : : :- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (19) + : : +- BroadcastQueryStage (17), Statistics(X) + : : +- ColumnarBroadcastExchange (16) + : : +- ^ ProjectExecTransformer (14) + : : +- ^ FilterExecTransformer (13) + : : +- ^ Scan parquet (12) + : +- ^ FilterExecTransformer (43) + : +- ^ ProjectExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ InputIteratorTransformer (40) + : +- ShuffleQueryStage (38), Statistics(X) + : +- ColumnarExchange (37) + : +- ^ ProjectExecTransformer (35) + : +- ^ FlushableHashAggregateExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) + : :- ^ ProjectExecTransformer (28) + : : +- ^ FilterExecTransformer (27) + : : +- ^ Scan parquet (26) + : +- ^ InputIteratorTransformer (32) + : +- BroadcastQueryStage (30), Statistics(X) + : +- ReusedExchange (29) +- ^ InputIteratorTransformer (60) - +- ^ InputAdapter (59) - +- ^ BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + +- BroadcastQueryStage (58), Statistics(X) + +- ColumnarBroadcastExchange (57) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ Scan parquet (53) +- == Initial Plan == Sort (103) +- Exchange (102) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt index 46792bf54410..252a06b11fa7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt @@ -4,54 +4,48 @@ AdaptiveSparkPlan (91) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (90) +- HashAggregate (89) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt index 38d759bb1020..5a6f0dc8fe05 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt @@ -4,25 +4,22 @@ AdaptiveSparkPlan (38) VeloxColumnarToRowExec (26) +- ^ SortExecTransformer (24) +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (37) +- Exchange (36) @@ -214,13 +211,12 @@ AdaptiveSparkPlan (57) VeloxColumnarToRowExec (50) +- ^ RegularHashAggregateExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == HashAggregate (56) +- Exchange (55) @@ -327,13 +323,12 @@ AdaptiveSparkPlan (57) VeloxColumnarToRowExec (50) +- ^ RegularHashAggregateExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == HashAggregate (56) +- Exchange (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 0b8395a482ea..722def52f850 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -6,31 +6,28 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (31) +- ^ RegularHashAggregateExecTransformer (30) +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == TakeOrderedAndProject (52) +- HashAggregate (51) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt index a9b7bac67fc4..0ca16a6174b7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt index fb55e067f7e9..24ba48c495f6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt @@ -4,60 +4,53 @@ AdaptiveSparkPlan (100) VeloxColumnarToRowExec (65) +- ^ SortExecTransformer (63) +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (99) +- Exchange (98) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt index 15f764040184..68854bdea473 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt index 4c62f3ce9132..fd1e1e8fa37f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt @@ -4,56 +4,49 @@ AdaptiveSparkPlan (93) VeloxColumnarToRowExec (60) +- ^ SortExecTransformer (58) +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == Sort (92) +- Exchange (91) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt index 5d4ef3143b4f..fc7a79a5b2b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt @@ -4,77 +4,68 @@ AdaptiveSparkPlan (129) VeloxColumnarToRowExec (84) +- ^ SortExecTransformer (82) +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == Sort (128) +- Exchange (127) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt index 227e77ecc700..4f18dfa35261 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt @@ -4,59 +4,52 @@ AdaptiveSparkPlan (98) VeloxColumnarToRowExec (64) +- ^ SortExecTransformer (62) +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (97) +- Exchange (96) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt index 634f26c86f24..090a9522f13a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index d6b77aaf9db7..808f58b58fa5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -6,39 +6,35 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (40) +- ^ RegularHashAggregateExecTransformer (39) +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + +- ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == TakeOrderedAndProject (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt index 8ee504fafda7..e6f8232f12c4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt @@ -4,36 +4,32 @@ AdaptiveSparkPlan (58) VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == Sort (57) +- Exchange (56) @@ -326,24 +322,21 @@ AdaptiveSparkPlan (99) +- ^ ProjectExecTransformer (80) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) + :- ^ ProjectExecTransformer (66) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) + : :- ^ FilterExecTransformer (60) + : : +- ^ Scan parquet (59) + : +- ^ InputIteratorTransformer (64) + : +- BroadcastQueryStage (62), Statistics(X) + : +- ReusedExchange (61) + +- ^ InputIteratorTransformer (70) + +- BroadcastQueryStage (68), Statistics(X) + +- ReusedExchange (67) +- == Initial Plan == HashAggregate (98) +- Exchange (97) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt index 55be10b8eb23..261e4fc821ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt @@ -4,27 +4,24 @@ AdaptiveSparkPlan (42) VeloxColumnarToRowExec (28) +- ^ SortExecTransformer (26) +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (41) +- Exchange (40) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt index efeb21b15e68..7ccf5dafe28e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt index f4f21eafc08a..8655ec75dd6f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt @@ -5,21 +5,19 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (33) +- Exchange (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt index baf2075cf671..0853cd6948f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt @@ -8,22 +8,20 @@ AdaptiveSparkPlan (41) +- ^ ProjectExecTransformer (21) +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (19) +- ^ RegularHashAggregateExecTransformer (18) +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (40) +- Exchange (39) @@ -239,14 +237,13 @@ AdaptiveSparkPlan (66) +- ^ ProjectExecTransformer (53) +- ^ RegularHashAggregateExecTransformer (52) +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ FilterExecTransformer (43) + +- ^ Scan parquet (42) +- == Initial Plan == HashAggregate (65) +- HashAggregate (64) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt index 13f5405e5091..78c65e72e6c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt @@ -4,33 +4,29 @@ AdaptiveSparkPlan (56) VeloxColumnarToRowExec (35) +- ^ SortExecTransformer (33) +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (55) +- Exchange (54) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt index 4a2b5a1744e0..62612096db42 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt @@ -5,14 +5,12 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (12) +- ^ RegularHashAggregateExecTransformer (11) +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- ^ InputAdapter (3) - +- ^ RowToVeloxColumnar (2) - +- ^ LocalTableScan (1) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == HashAggregate (34) +- Exchange (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 09ceccba8b0a..58751c9b885f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -5,49 +5,43 @@ AdaptiveSparkPlan (86) +- TakeOrderedAndProjectExecTransformer (52) +- ^ RegularHashAggregateExecTransformer (50) +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + +- ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) + :- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == TakeOrderedAndProject (85) +- HashAggregate (84) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt index ded467ae7006..e2f0c051b69a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt @@ -4,21 +4,19 @@ AdaptiveSparkPlan (33) VeloxColumnarToRowExec (21) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == HashAggregate (32) +- Exchange (31) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt index e1ce25665b3e..3b7e5334a3ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt @@ -12,49 +12,43 @@ AdaptiveSparkPlan (96) : :- ^ FilterExecTransformer (2) : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- ^ InputAdapter (17) - : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ FilterExecTransformer (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- ^ InputAdapter (11) - : : +- ^ BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ FilterExecTransformer (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ^ InputAdapter (32) - : +- ^ ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) + : +- BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ FilterExecTransformer (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ FilterExecTransformer (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + +- BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == Sort (95) +- Exchange (94) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt index 31a2b07b31f9..f81832ea4c1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt @@ -4,54 +4,48 @@ AdaptiveSparkPlan (91) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (90) +- HashAggregate (89) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt index bfc52b25c52a..5dac875505ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt @@ -4,25 +4,22 @@ AdaptiveSparkPlan (38) VeloxColumnarToRowExec (26) +- ^ SortExecTransformer (24) +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + +- ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == Sort (37) +- Exchange (36) @@ -216,13 +213,12 @@ AdaptiveSparkPlan (57) VeloxColumnarToRowExec (50) +- ^ RegularHashAggregateExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == HashAggregate (56) +- Exchange (55) @@ -329,13 +325,12 @@ AdaptiveSparkPlan (57) VeloxColumnarToRowExec (50) +- ^ RegularHashAggregateExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == HashAggregate (56) +- Exchange (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index fdde00c0111b..7ee3b1a0cfd4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -6,31 +6,28 @@ AdaptiveSparkPlan (53) +- ^ ProjectExecTransformer (31) +- ^ RegularHashAggregateExecTransformer (30) +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == TakeOrderedAndProject (52) +- HashAggregate (51) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt index a423034982a9..65a268b41109 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt @@ -4,28 +4,25 @@ AdaptiveSparkPlan (44) VeloxColumnarToRowExec (29) +- ^ SortExecTransformer (27) +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == Sort (43) +- Exchange (42) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt index 6ce78c899186..a86ee299c7c3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt @@ -4,60 +4,53 @@ AdaptiveSparkPlan (100) VeloxColumnarToRowExec (65) +- ^ SortExecTransformer (63) +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (99) +- Exchange (98) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt index 0882ff9e151c..12d6c3ea85e4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt index 3a43310c0813..571b6c13b6fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt @@ -4,56 +4,49 @@ AdaptiveSparkPlan (93) VeloxColumnarToRowExec (60) +- ^ SortExecTransformer (58) +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == Sort (92) +- Exchange (91) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt index 97a1de2e4861..af98ad739c21 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt @@ -4,77 +4,68 @@ AdaptiveSparkPlan (129) VeloxColumnarToRowExec (84) +- ^ SortExecTransformer (82) +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == Sort (128) +- Exchange (127) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt index 8003b4e2a7c0..c88eba3bb12e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt @@ -4,59 +4,52 @@ AdaptiveSparkPlan (98) VeloxColumnarToRowExec (64) +- ^ SortExecTransformer (62) +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == Sort (97) +- Exchange (96) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 8d67aad16c3d..9316f6f8ebb5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index a545d26e2b12..a09e6167ffd3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -6,53 +6,46 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (57) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (86) +- HashAggregate (85) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 717acdacccd4..47b241e9a343 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -4,46 +4,40 @@ AdaptiveSparkPlan (72) VeloxColumnarToRowExec (50) +- ^ SortExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (45) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == Sort (71) +- Exchange (70) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index ff48c00b1d85..9adf1b6060ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (49) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (48) +- Exchange (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index 48e48c97477f..fb354643ede9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index c4893bc66912..62a2d4a7b617 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -8,19 +8,17 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (34) +- HashAggregate (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 03aaba455979..791ededdabda 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -4,29 +4,26 @@ AdaptiveSparkPlan (45) VeloxColumnarToRowExec (30) +- ^ SortExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (44) +- Exchange (43) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index be28ff7e84eb..2060266a2550 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -4,39 +4,34 @@ AdaptiveSparkPlan (64) VeloxColumnarToRowExec (42) +- ^ SortExecTransformer (40) +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (63) +- Exchange (62) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index e2ff1fcc0eb5..504ee433e9e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -10,30 +10,27 @@ AdaptiveSparkPlan (57) :- ^ ProjectExecTransformer (18) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) +- ^ FilterExecTransformer (30) +- ^ ProjectExecTransformer (29) +- ^ RegularHashAggregateExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == HashAggregate (56) +- HashAggregate (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 3b6719e7e55d..78efb8c67470 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -8,56 +8,49 @@ AdaptiveSparkPlan (97) +- ^ ProjectExecTransformer (59) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) + : +- ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) +- ^ ProjectExecTransformer (56) +- ^ FilterExecTransformer (55) +- ^ RegularHashAggregateExecTransformer (54) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51) - +- ReusedExchange (50) + +- ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == TakeOrderedAndProject (96) +- HashAggregate (95) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index 290d398f4edd..aa13d1509c29 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -7,19 +7,17 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (33) +- HashAggregate (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index bdf5d16a8fd3..959bcd4ef703 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -4,78 +4,67 @@ AdaptiveSparkPlan (126) VeloxColumnarToRowExec (86) +- ^ SortExecTransformer (84) +- ^ InputIteratorTransformer (83) - +- ^ InputAdapter (82) - +- ^ ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (125) +- Exchange (124) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index b960a7682beb..e9418fc2a71c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -5,73 +5,63 @@ AdaptiveSparkPlan (119) +- TakeOrderedAndProjectExecTransformer (81) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == TakeOrderedAndProject (118) +- HashAggregate (117) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 67b2d945c059..2b0fcd16aadc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -4,31 +4,27 @@ AdaptiveSparkPlan (46) VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (45) +- Exchange (44) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index 6061266d1ab3..ed7b3ea7d377 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -9,32 +9,28 @@ AdaptiveSparkPlan (59) +- ^ ProjectExecTransformer (33) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == TakeOrderedAndProject (58) +- HashAggregate (57) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index be03c6970a0a..07a00316284a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (50) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (49) +- Exchange (48) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index a7093229bb11..766f9f5a1314 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (134) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (133) +- Exchange (132) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index 8d1a71e9751e..7aae3dccfd9b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 132fd068b4d5..67c25166ae14 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -4,81 +4,69 @@ AdaptiveSparkPlan (128) VeloxColumnarToRowExec (90) +- ^ SortExecTransformer (88) +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70) - +- ReusedExchange (69) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70) + +- ReusedExchange (69) +- == Initial Plan == Sort (127) +- Exchange (126) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 043826c912d8..4c1fb04b8660 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -4,111 +4,95 @@ AdaptiveSparkPlan (177) VeloxColumnarToRowExec (125) +- ^ SortExecTransformer (123) +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + +- ShuffleQueryStage (120) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == Sort (176) +- Exchange (175) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index bc8752ca1f2d..582d323dccb2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (133) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (132) +- Exchange (131) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index 74f79bd3ee64..63b30bb5d26b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 5050dab789f0..6213f61ffdf4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -6,53 +6,46 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (57) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (86) +- HashAggregate (85) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 31ab69e36d50..80dbdb2c50ad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -4,46 +4,40 @@ AdaptiveSparkPlan (72) VeloxColumnarToRowExec (50) +- ^ SortExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == Sort (71) +- Exchange (70) @@ -389,26 +383,22 @@ AdaptiveSparkPlan (120) +- ^ ProjectExecTransformer (97) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ^ InputAdapter (79) - : : +- ^ ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) + : :- ^ InputIteratorTransformer (80) + : : +- ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ NoopFilter (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) +- ^ InputIteratorTransformer (95) - +- ^ InputAdapter (94) - +- ^ ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + +- ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == HashAggregate (119) +- HashAggregate (118) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index 7f4249f58548..58a8788ca252 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (49) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (48) +- Exchange (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 7ba31590eb06..8837c6ef4143 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 379450102787..2c06906d4fb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -8,19 +8,17 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (34) +- HashAggregate (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index cb25386ec7a4..270cea0f6f46 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -8,23 +8,21 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (22) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (41) +- Exchange (40) @@ -242,14 +240,13 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (54) +- ^ RegularHashAggregateExecTransformer (53) +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == HashAggregate (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index f6613670b1da..90c77f0de9b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -4,39 +4,34 @@ AdaptiveSparkPlan (64) VeloxColumnarToRowExec (42) +- ^ SortExecTransformer (40) +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (63) +- Exchange (62) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index d8ec36fbe69f..16321ffa4e59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -10,30 +10,27 @@ AdaptiveSparkPlan (57) :- ^ ProjectExecTransformer (18) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) +- ^ FilterExecTransformer (30) +- ^ ProjectExecTransformer (29) +- ^ RegularHashAggregateExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == HashAggregate (56) +- HashAggregate (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index 1250303cf3b9..fa441bac82cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -8,56 +8,49 @@ AdaptiveSparkPlan (97) +- ^ ProjectExecTransformer (59) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) +- ^ ProjectExecTransformer (56) +- ^ FilterExecTransformer (55) +- ^ RegularHashAggregateExecTransformer (54) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + +- ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == TakeOrderedAndProject (96) +- HashAggregate (95) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index 72aafd6a63af..d9ceedf0db6c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -7,19 +7,17 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (33) +- HashAggregate (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index 229e17b97d3f..305f98339a44 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -8,72 +8,62 @@ AdaptiveSparkPlan (123) +- ^ ProjectExecTransformer (78) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (122) +- Exchange (121) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index 924e087f8ac2..04cc1f47a3d3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -4,73 +4,63 @@ AdaptiveSparkPlan (118) VeloxColumnarToRowExec (81) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == TakeOrderedAndProject (117) +- HashAggregate (116) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index a40eb7e2b0c2..9a513e223197 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -4,31 +4,27 @@ AdaptiveSparkPlan (46) VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (45) +- Exchange (44) @@ -251,13 +247,12 @@ AdaptiveSparkPlan (65) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == HashAggregate (64) +- Exchange (63) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 4c1028c2e089..0b11cfa1f763 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -9,32 +9,28 @@ AdaptiveSparkPlan (59) +- ^ ProjectExecTransformer (33) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == TakeOrderedAndProject (58) +- HashAggregate (57) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index b0cd269ae184..2eb1f1044104 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (50) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (49) +- Exchange (48) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 082197eda475..85b46c76a499 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (134) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (133) +- Exchange (132) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index 51c5836bdd11..e03830992c2e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index 60738454f20a..a2e4435d7188 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -4,81 +4,69 @@ AdaptiveSparkPlan (128) VeloxColumnarToRowExec (90) +- ^ SortExecTransformer (88) +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == Sort (127) +- Exchange (126) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 5549da0cecd2..47be7c7230dd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -4,111 +4,95 @@ AdaptiveSparkPlan (177) VeloxColumnarToRowExec (125) +- ^ SortExecTransformer (123) +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + +- ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == Sort (176) +- Exchange (175) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index 2925cced3f2e..93001ea6c78d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (133) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (132) +- Exchange (131) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index 41613c36f7bd..c254ec8c82ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 6c98f7aba99d..43930a2eb5db 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -6,53 +6,46 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (57) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (86) +- HashAggregate (85) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index 59eee0048de4..e8e0cb750c84 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -4,46 +4,40 @@ AdaptiveSparkPlan (72) VeloxColumnarToRowExec (50) +- ^ SortExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == Sort (71) +- Exchange (70) @@ -393,26 +387,22 @@ AdaptiveSparkPlan (120) +- ^ ProjectExecTransformer (97) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ^ InputAdapter (79) - : : +- ^ ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) + : :- ^ InputIteratorTransformer (80) + : : +- ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ NoopFilter (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) +- ^ InputIteratorTransformer (95) - +- ^ InputAdapter (94) - +- ^ ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + +- ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == HashAggregate (119) +- HashAggregate (118) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index 1e676f1c8275..ea49c8c6402c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (49) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (48) +- Exchange (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 87ea62a744f3..a7a55ece84ee 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index d118caee8ae4..412a680dae78 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -8,19 +8,17 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (34) +- HashAggregate (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index 6b5d089f4580..c66afda3aee4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -8,23 +8,21 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (22) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (41) +- Exchange (40) @@ -244,14 +242,13 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (54) +- ^ RegularHashAggregateExecTransformer (53) +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == HashAggregate (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 57a1fcfd1bce..7bb61df21db4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -4,39 +4,34 @@ AdaptiveSparkPlan (64) VeloxColumnarToRowExec (42) +- ^ SortExecTransformer (40) +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (63) +- Exchange (62) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index 03afc1f85103..c51280b11b0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -10,30 +10,27 @@ AdaptiveSparkPlan (57) :- ^ ProjectExecTransformer (18) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) +- ^ FilterExecTransformer (30) +- ^ ProjectExecTransformer (29) +- ^ RegularHashAggregateExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == HashAggregate (56) +- HashAggregate (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 598b86c342e9..0f3bc73eb825 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -8,56 +8,49 @@ AdaptiveSparkPlan (97) +- ^ ProjectExecTransformer (59) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) +- ^ ProjectExecTransformer (56) +- ^ FilterExecTransformer (55) +- ^ RegularHashAggregateExecTransformer (54) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + +- ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == TakeOrderedAndProject (96) +- HashAggregate (95) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 2b5d5b20ea54..1d1169055f32 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -7,19 +7,17 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (33) +- HashAggregate (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index ab7fa3fe53c9..71c30b25fb2b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -8,72 +8,62 @@ AdaptiveSparkPlan (123) +- ^ ProjectExecTransformer (78) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (122) +- Exchange (121) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 5c9ac1690ea4..d9d79a967f03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -4,73 +4,63 @@ AdaptiveSparkPlan (118) VeloxColumnarToRowExec (81) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == TakeOrderedAndProject (117) +- HashAggregate (116) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 12dbaf0baf75..feeda6c9fded 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -4,31 +4,27 @@ AdaptiveSparkPlan (46) VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (45) +- Exchange (44) @@ -253,13 +249,12 @@ AdaptiveSparkPlan (65) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == HashAggregate (64) +- Exchange (63) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index c3de65c763ca..ee94233a0a5b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -9,32 +9,28 @@ AdaptiveSparkPlan (59) +- ^ ProjectExecTransformer (33) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == TakeOrderedAndProject (58) +- HashAggregate (57) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index ab8ecadcb532..141bd8aa73fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (50) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (49) +- Exchange (48) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 3296a70a70f5..95a5f3ee722f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (134) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (133) +- Exchange (132) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 9333e2a8ad59..5987a808f5fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 4a641f05b878..994d1e163a36 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -4,81 +4,69 @@ AdaptiveSparkPlan (128) VeloxColumnarToRowExec (90) +- ^ SortExecTransformer (88) +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == Sort (127) +- Exchange (126) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index cf125fb93008..b483283286d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -4,111 +4,95 @@ AdaptiveSparkPlan (177) VeloxColumnarToRowExec (125) +- ^ SortExecTransformer (123) +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + +- ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == Sort (176) +- Exchange (175) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 426fb391c048..5d1350564eac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (133) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (132) +- Exchange (131) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt index 7a80ed95f3e8..656f98574483 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 5c3e1a93c499..313572f951ad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -6,53 +6,46 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (57) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ FilterExecTransformer (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (86) +- HashAggregate (85) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index 187df108a32a..e91064e3580f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -4,46 +4,40 @@ AdaptiveSparkPlan (72) VeloxColumnarToRowExec (50) +- ^ SortExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (45) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == Sort (71) +- Exchange (70) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index 2df3770552eb..8bad82473a58 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (49) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (48) +- Exchange (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index 64fd78ed05c9..3268390701fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt index 45951bf4c41e..dd4a53cfa822 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt @@ -8,19 +8,17 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (34) +- HashAggregate (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index 2576137dbd7b..c456274bfeaa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -4,29 +4,26 @@ AdaptiveSparkPlan (45) VeloxColumnarToRowExec (30) +- ^ SortExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (44) +- Exchange (43) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index 875d74780095..7bbe4249fdcd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -4,39 +4,34 @@ AdaptiveSparkPlan (64) VeloxColumnarToRowExec (42) +- ^ SortExecTransformer (40) +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (63) +- Exchange (62) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index e3a6142c740b..248f638db9ad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -10,30 +10,27 @@ AdaptiveSparkPlan (57) :- ^ ProjectExecTransformer (18) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) +- ^ FilterExecTransformer (30) +- ^ ProjectExecTransformer (29) +- ^ RegularHashAggregateExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ FilterExecTransformer (20) + +- ^ Scan parquet (19) +- == Initial Plan == HashAggregate (56) +- HashAggregate (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index b1f06863e2ca..428186218984 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -8,56 +8,49 @@ AdaptiveSparkPlan (97) +- ^ ProjectExecTransformer (59) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) + : +- ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ FilterExecTransformer (43) + : +- ^ Scan parquet (42) +- ^ ProjectExecTransformer (56) +- ^ FilterExecTransformer (55) +- ^ RegularHashAggregateExecTransformer (54) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51) - +- ReusedExchange (50) + +- ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == TakeOrderedAndProject (96) +- HashAggregate (95) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt index fe7cc6d3a2e8..4ea9d09e13f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt @@ -7,19 +7,17 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (33) +- HashAggregate (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt index 6e2327ac1336..4fe956080086 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt @@ -4,78 +4,67 @@ AdaptiveSparkPlan (126) VeloxColumnarToRowExec (86) +- ^ SortExecTransformer (84) +- ^ InputIteratorTransformer (83) - +- ^ InputAdapter (82) - +- ^ ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ FilterExecTransformer (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ FilterExecTransformer (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ FilterExecTransformer (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (125) +- Exchange (124) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index 0a51e3da621c..6f94bd5c8d01 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -5,73 +5,63 @@ AdaptiveSparkPlan (119) +- TakeOrderedAndProjectExecTransformer (81) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ FilterExecTransformer (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ FilterExecTransformer (63) + +- ^ Scan parquet (62) +- == Initial Plan == TakeOrderedAndProject (118) +- HashAggregate (117) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index bc7ca6a0ae16..77c3b584125b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -4,31 +4,27 @@ AdaptiveSparkPlan (46) VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (45) +- Exchange (44) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index 80da568524e6..6e4f9178de19 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -9,32 +9,28 @@ AdaptiveSparkPlan (59) +- ^ ProjectExecTransformer (33) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == TakeOrderedAndProject (58) +- HashAggregate (57) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index 02f119e952b1..d2e7a9cffbd6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (50) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (49) +- Exchange (48) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index d8b833813def..aff82bdec961 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (134) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (133) +- Exchange (132) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt index fa2536a365c8..b39d8c1b2aec 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index 77f586cd8897..f0650d65e2cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -4,81 +4,69 @@ AdaptiveSparkPlan (128) VeloxColumnarToRowExec (90) +- ^ SortExecTransformer (88) +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70) - +- ReusedExchange (69) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70) + +- ReusedExchange (69) +- == Initial Plan == Sort (127) +- Exchange (126) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index 845d1ac43a77..e2a1907549f0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -4,111 +4,95 @@ AdaptiveSparkPlan (177) VeloxColumnarToRowExec (125) +- ^ SortExecTransformer (123) +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + +- ShuffleQueryStage (120) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ FilterExecTransformer (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ FilterExecTransformer (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ FilterExecTransformer (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ FilterExecTransformer (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ FilterExecTransformer (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ FilterExecTransformer (100) + +- ^ Scan parquet (99) +- == Initial Plan == Sort (176) +- Exchange (175) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index 239cda5088bf..b86e86d560e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (133) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (132) +- Exchange (131) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt index 0cacd70fdca7..89de3133895b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index 11912c5ca02b..680971de4c1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -6,53 +6,46 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (57) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ FilterExecTransformer (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (86) +- HashAggregate (85) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index 6180b1966357..b964ec1ed8d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -4,46 +4,40 @@ AdaptiveSparkPlan (72) VeloxColumnarToRowExec (50) +- ^ SortExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == Sort (71) +- Exchange (70) @@ -389,26 +383,22 @@ AdaptiveSparkPlan (120) +- ^ ProjectExecTransformer (97) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ^ InputAdapter (79) - : : +- ^ ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) + : :- ^ InputIteratorTransformer (80) + : : +- ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ FilterExecTransformer (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) +- ^ InputIteratorTransformer (95) - +- ^ InputAdapter (94) - +- ^ ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + +- ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == HashAggregate (119) +- HashAggregate (118) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index fad8bed52e6a..fabde3ecd687 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (49) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (48) +- Exchange (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index c10d12cacb71..6ca6e75ac545 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt index 222f6d64a5e7..c1a4a2fe884f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt @@ -8,19 +8,17 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (34) +- HashAggregate (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index 95971f8d36a7..a45c61431782 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -8,23 +8,21 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (22) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (41) +- Exchange (40) @@ -242,14 +240,13 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (54) +- ^ RegularHashAggregateExecTransformer (53) +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ FilterExecTransformer (44) + +- ^ Scan parquet (43) +- == Initial Plan == HashAggregate (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index 39563ad3bf98..9a927b77805e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -4,39 +4,34 @@ AdaptiveSparkPlan (64) VeloxColumnarToRowExec (42) +- ^ SortExecTransformer (40) +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (63) +- Exchange (62) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 1342964d7f91..48ba7ddab093 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -10,30 +10,27 @@ AdaptiveSparkPlan (57) :- ^ ProjectExecTransformer (18) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) +- ^ FilterExecTransformer (30) +- ^ ProjectExecTransformer (29) +- ^ RegularHashAggregateExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ FilterExecTransformer (20) + +- ^ Scan parquet (19) +- == Initial Plan == HashAggregate (56) +- HashAggregate (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index d45968ecc14a..a15a185bbd77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -8,56 +8,49 @@ AdaptiveSparkPlan (97) +- ^ ProjectExecTransformer (59) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ FilterExecTransformer (43) + : +- ^ Scan parquet (42) +- ^ ProjectExecTransformer (56) +- ^ FilterExecTransformer (55) +- ^ RegularHashAggregateExecTransformer (54) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + +- ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == TakeOrderedAndProject (96) +- HashAggregate (95) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt index bdb709493df4..80f0189d5d65 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt @@ -7,19 +7,17 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (33) +- HashAggregate (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt index 72107db4e377..a0dcc7029a35 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt @@ -8,72 +8,62 @@ AdaptiveSparkPlan (123) +- ^ ProjectExecTransformer (78) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ FilterExecTransformer (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ FilterExecTransformer (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ FilterExecTransformer (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (122) +- Exchange (121) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index b53c6c158fe9..3635363cfe47 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -4,73 +4,63 @@ AdaptiveSparkPlan (118) VeloxColumnarToRowExec (81) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ FilterExecTransformer (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ FilterExecTransformer (63) + +- ^ Scan parquet (62) +- == Initial Plan == TakeOrderedAndProject (117) +- HashAggregate (116) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index c63d8516f6a6..3c18ab436ed2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -4,31 +4,27 @@ AdaptiveSparkPlan (46) VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (45) +- Exchange (44) @@ -251,13 +247,12 @@ AdaptiveSparkPlan (65) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ Scan parquet (47) +- == Initial Plan == HashAggregate (64) +- Exchange (63) @@ -364,13 +359,12 @@ AdaptiveSparkPlan (65) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ Scan parquet (47) +- == Initial Plan == HashAggregate (64) +- Exchange (63) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index 9461041ef2b0..eebd274d4709 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -9,32 +9,28 @@ AdaptiveSparkPlan (59) +- ^ ProjectExecTransformer (33) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == TakeOrderedAndProject (58) +- HashAggregate (57) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index 3c4e85011fd3..f05c9a5378c6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (50) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (49) +- Exchange (48) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 677e44e3e6a8..027ea4a926d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (134) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (133) +- Exchange (132) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt index 15f764040184..68854bdea473 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index 8a8f2442f25c..eb5979fb5d84 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -4,81 +4,69 @@ AdaptiveSparkPlan (128) VeloxColumnarToRowExec (90) +- ^ SortExecTransformer (88) +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == Sort (127) +- Exchange (126) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index 2e6e8038f633..98bba133502c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -4,111 +4,95 @@ AdaptiveSparkPlan (177) VeloxColumnarToRowExec (125) +- ^ SortExecTransformer (123) +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + +- ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ FilterExecTransformer (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ FilterExecTransformer (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ FilterExecTransformer (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ FilterExecTransformer (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ FilterExecTransformer (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ FilterExecTransformer (100) + +- ^ Scan parquet (99) +- == Initial Plan == Sort (176) +- Exchange (175) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 48b91754df6c..7f3917f80457 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (133) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (132) +- Exchange (131) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt index 634f26c86f24..090a9522f13a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt @@ -4,19 +4,17 @@ AdaptiveSparkPlan (28) VeloxColumnarToRowExec (19) +- ^ SortExecTransformer (17) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == Sort (27) +- Exchange (26) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 869bb7de0e36..02cf374dd013 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -6,53 +6,46 @@ AdaptiveSparkPlan (87) +- ^ ProjectExecTransformer (57) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ FilterExecTransformer (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == TakeOrderedAndProject (86) +- HashAggregate (85) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index 494978555e23..b14866370aab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -4,46 +4,40 @@ AdaptiveSparkPlan (72) VeloxColumnarToRowExec (50) +- ^ SortExecTransformer (48) +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == Sort (71) +- Exchange (70) @@ -393,26 +387,22 @@ AdaptiveSparkPlan (120) +- ^ ProjectExecTransformer (97) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ^ InputAdapter (79) - : : +- ^ ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) + : :- ^ InputIteratorTransformer (80) + : : +- ShuffleQueryStage (78), Statistics(X) + : : +- ColumnarExchange (77) + : : +- ^ ProjectExecTransformer (75) + : : +- ^ FilterExecTransformer (74) + : : +- ^ Scan parquet (73) + : +- ^ InputIteratorTransformer (84) + : +- ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) +- ^ InputIteratorTransformer (95) - +- ^ InputAdapter (94) - +- ^ ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + +- ShuffleQueryStage (93), Statistics(X) + +- ReusedExchange (92) +- == Initial Plan == HashAggregate (119) +- HashAggregate (118) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 6e4c7befbe03..27765da815dd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (49) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (48) +- Exchange (47) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index ab591d0dceee..79b9dcd18bd1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -4,34 +4,30 @@ AdaptiveSparkPlan (52) VeloxColumnarToRowExec (36) +- ^ SortExecTransformer (34) +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == Sort (51) +- Exchange (50) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt index 76297b4084f8..4db67eada562 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt @@ -8,19 +8,17 @@ AdaptiveSparkPlan (35) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (34) +- HashAggregate (33) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index 88f14fb60ce6..c8f4e2c84ac3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -8,23 +8,21 @@ AdaptiveSparkPlan (42) +- ^ ProjectExecTransformer (22) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ FilterExecTransformer (20) +- ^ RegularHashAggregateExecTransformer (19) +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (41) +- Exchange (40) @@ -244,14 +242,13 @@ AdaptiveSparkPlan (67) +- ^ ProjectExecTransformer (54) +- ^ RegularHashAggregateExecTransformer (53) +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ FilterExecTransformer (44) + +- ^ Scan parquet (43) +- == Initial Plan == HashAggregate (66) +- HashAggregate (65) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index cc862d451070..247b853fb1c4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -4,39 +4,34 @@ AdaptiveSparkPlan (64) VeloxColumnarToRowExec (42) +- ^ SortExecTransformer (40) +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (63) +- Exchange (62) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index a49104ee6fc6..fad48184fed1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -10,30 +10,27 @@ AdaptiveSparkPlan (57) :- ^ ProjectExecTransformer (18) : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) +- ^ FilterExecTransformer (30) +- ^ ProjectExecTransformer (29) +- ^ RegularHashAggregateExecTransformer (28) +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ FilterExecTransformer (20) + +- ^ Scan parquet (19) +- == Initial Plan == HashAggregate (56) +- HashAggregate (55) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index c599a4c246b3..5a6f04064349 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -8,56 +8,49 @@ AdaptiveSparkPlan (97) +- ^ ProjectExecTransformer (59) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ FilterExecTransformer (43) + : +- ^ Scan parquet (42) +- ^ ProjectExecTransformer (56) +- ^ FilterExecTransformer (55) +- ^ RegularHashAggregateExecTransformer (54) +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + +- ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == TakeOrderedAndProject (96) +- HashAggregate (95) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt index 214230b1bba6..2f7bb7995dd9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt @@ -7,19 +7,17 @@ AdaptiveSparkPlan (34) +- ^ ProjectExecTransformer (18) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == HashAggregate (33) +- HashAggregate (32) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt index 85eafca643b9..e8aa97a29e7a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt @@ -8,72 +8,62 @@ AdaptiveSparkPlan (123) +- ^ ProjectExecTransformer (78) +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ FilterExecTransformer (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ FilterExecTransformer (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ FilterExecTransformer (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (122) +- Exchange (121) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index 04a1a76967b9..323db548d24e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -4,73 +4,63 @@ AdaptiveSparkPlan (118) VeloxColumnarToRowExec (81) +- ^ RegularHashAggregateExecTransformer (79) +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ FilterExecTransformer (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ FilterExecTransformer (63) + +- ^ Scan parquet (62) +- == Initial Plan == TakeOrderedAndProject (117) +- HashAggregate (116) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index 864c9a3d40a4..add978887b0a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -4,31 +4,27 @@ AdaptiveSparkPlan (46) VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (45) +- Exchange (44) @@ -253,13 +249,12 @@ AdaptiveSparkPlan (65) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ Scan parquet (47) +- == Initial Plan == HashAggregate (64) +- Exchange (63) @@ -366,13 +361,12 @@ AdaptiveSparkPlan (65) VeloxColumnarToRowExec (58) +- ^ RegularHashAggregateExecTransformer (56) +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ Scan parquet (47) +- == Initial Plan == HashAggregate (64) +- Exchange (63) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index d25da8196eb2..69dc65c58e21 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -9,32 +9,28 @@ AdaptiveSparkPlan (59) +- ^ ProjectExecTransformer (33) +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == TakeOrderedAndProject (58) +- HashAggregate (57) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index 43af17cfcc73..d7f42dd7b351 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -4,32 +4,28 @@ AdaptiveSparkPlan (50) VeloxColumnarToRowExec (34) +- ^ SortExecTransformer (32) +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == Sort (49) +- Exchange (48) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index dd2a2cc31a75..129b02c9548e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (134) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (133) +- Exchange (132) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt index 0882ff9e151c..12d6c3ea85e4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt @@ -4,13 +4,12 @@ AdaptiveSparkPlan (19) VeloxColumnarToRowExec (12) +- ^ RegularHashAggregateExecTransformer (10) +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + +- ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == HashAggregate (18) +- Exchange (17) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index 32199dcf8e1b..1e6af0683a39 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -4,81 +4,69 @@ AdaptiveSparkPlan (128) VeloxColumnarToRowExec (90) +- ^ SortExecTransformer (88) +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == Sort (127) +- Exchange (126) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index e30d96500f74..cfc03953fe18 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -4,111 +4,95 @@ AdaptiveSparkPlan (177) VeloxColumnarToRowExec (125) +- ^ SortExecTransformer (123) +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + +- ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ FilterExecTransformer (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ FilterExecTransformer (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ FilterExecTransformer (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ FilterExecTransformer (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ FilterExecTransformer (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ FilterExecTransformer (100) + +- ^ Scan parquet (99) +- == Initial Plan == Sort (176) +- Exchange (175) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index 4f61f99709bb..9a8c9a87aac2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -4,84 +4,72 @@ AdaptiveSparkPlan (133) VeloxColumnarToRowExec (94) +- ^ SortExecTransformer (92) +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ FilterExecTransformer (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ FilterExecTransformer (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ FilterExecTransformer (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FilterExecTransformer (70) + +- ^ Scan parquet (69) +- == Initial Plan == Sort (132) +- Exchange (131) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala index 7dfa0563d743..a49e8aa518b6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala @@ -112,7 +112,7 @@ trait UnaryTransformSupport extends TransformSupport with UnaryExecNode { case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = false)( val transformStageId: Int -) extends GenerateTreeStringShim +) extends WholeStageTransformerGenerateTreeStringShim with UnaryTransformSupport { assert(child.isInstanceOf[TransformSupport]) diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala index c9bbf4e1c0eb..e5925e3ac4d0 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala @@ -160,7 +160,7 @@ case class ColumnarCollapseTransformStages( } case class ColumnarInputAdapter(child: SparkPlan) - extends UnaryExecNode + extends InputAdapterGenerateTreeStringShim with Convention.KnownBatchType { override def output: Seq[Attribute] = child.output override def supportsColumnar: Boolean = true diff --git a/shims/spark32/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala b/shims/spark32/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala index 3e42244bd8c6..8936e6ca6351 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} +import org.apache.spark.sql.execution.UnaryExecNode /** * Spark 3.5 has changed the parameter type of the generateTreeString API in TreeNode. In order to @@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} * allow different Spark versions to override their own generateTreeString. */ -trait GenerateTreeStringShim extends UnaryExecNode { +trait WholeStageTransformerGenerateTreeStringShim extends UnaryExecNode { def stageId: Int @@ -62,3 +62,28 @@ trait GenerateTreeStringShim extends UnaryExecNode { } } } + +trait InputAdapterGenerateTreeStringShim extends UnaryExecNode { + + override def generateTreeString( + depth: Int, + lastChildren: Seq[Boolean], + append: String => Unit, + verbose: Boolean, + prefix: String = "", + addSuffix: Boolean = false, + maxFields: Int, + printNodeId: Boolean, + indent: Int = 0): Unit = { + child.generateTreeString( + depth, + lastChildren, + append, + verbose, + prefix = "", + addSuffix = false, + maxFields, + printNodeId, + indent) + } +} diff --git a/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala b/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala index 3e42244bd8c6..8936e6ca6351 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} +import org.apache.spark.sql.execution.UnaryExecNode /** * Spark 3.5 has changed the parameter type of the generateTreeString API in TreeNode. In order to @@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} * allow different Spark versions to override their own generateTreeString. */ -trait GenerateTreeStringShim extends UnaryExecNode { +trait WholeStageTransformerGenerateTreeStringShim extends UnaryExecNode { def stageId: Int @@ -62,3 +62,28 @@ trait GenerateTreeStringShim extends UnaryExecNode { } } } + +trait InputAdapterGenerateTreeStringShim extends UnaryExecNode { + + override def generateTreeString( + depth: Int, + lastChildren: Seq[Boolean], + append: String => Unit, + verbose: Boolean, + prefix: String = "", + addSuffix: Boolean = false, + maxFields: Int, + printNodeId: Boolean, + indent: Int = 0): Unit = { + child.generateTreeString( + depth, + lastChildren, + append, + verbose, + prefix = "", + addSuffix = false, + maxFields, + printNodeId, + indent) + } +} diff --git a/shims/spark34/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala b/shims/spark34/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala index 3e42244bd8c6..8936e6ca6351 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} +import org.apache.spark.sql.execution.UnaryExecNode /** * Spark 3.5 has changed the parameter type of the generateTreeString API in TreeNode. In order to @@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} * allow different Spark versions to override their own generateTreeString. */ -trait GenerateTreeStringShim extends UnaryExecNode { +trait WholeStageTransformerGenerateTreeStringShim extends UnaryExecNode { def stageId: Int @@ -62,3 +62,28 @@ trait GenerateTreeStringShim extends UnaryExecNode { } } } + +trait InputAdapterGenerateTreeStringShim extends UnaryExecNode { + + override def generateTreeString( + depth: Int, + lastChildren: Seq[Boolean], + append: String => Unit, + verbose: Boolean, + prefix: String = "", + addSuffix: Boolean = false, + maxFields: Int, + printNodeId: Boolean, + indent: Int = 0): Unit = { + child.generateTreeString( + depth, + lastChildren, + append, + verbose, + prefix = "", + addSuffix = false, + maxFields, + printNodeId, + indent) + } +} diff --git a/shims/spark35/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala b/shims/spark35/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala index 1dc6d7f7174d..7628b210f1c1 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/execution/GenerateTreeStringShim.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} +import org.apache.spark.sql.execution.UnaryExecNode /** * Spark 3.5 has changed the parameter type of the generateTreeString API in TreeNode. In order to @@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} * allow different Spark versions to override their own generateTreeString. */ -trait GenerateTreeStringShim extends UnaryExecNode { +trait WholeStageTransformerGenerateTreeStringShim extends UnaryExecNode { def stageId: Int @@ -62,3 +62,28 @@ trait GenerateTreeStringShim extends UnaryExecNode { } } } + +trait InputAdapterGenerateTreeStringShim extends UnaryExecNode { + + override def generateTreeString( + depth: Int, + lastChildren: java.util.ArrayList[Boolean], + append: String => Unit, + verbose: Boolean, + prefix: String = "", + addSuffix: Boolean = false, + maxFields: Int, + printNodeId: Boolean, + indent: Int = 0): Unit = { + child.generateTreeString( + depth, + lastChildren, + append, + verbose, + prefix = "", + addSuffix = false, + maxFields, + printNodeId, + indent) + } +} From a36d121a897df02f24596fcc47fb25a16e29ae9f Mon Sep 17 00:00:00 2001 From: JiaKe Date: Thu, 6 Jun 2024 08:56:12 +0800 Subject: [PATCH 215/402] [GLUTEN-5720][VL] Enable left and right semi join type in smj (#5825) --- .../org/apache/gluten/execution/TestOperator.scala | 10 ++++++++++ cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index bc51ee7cb670..cd1f21a0a31c 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1246,6 +1246,16 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + withSQLConf("spark.gluten.sql.columnar.forceShuffledHashJoin" -> "false") { + runQueryAndCompare( + """ + |select * from t1 left semi join t2 on t1.c1 = t2.c1 and t1.c1 > 50; + |""".stripMargin + ) { + checkGlutenOperatorMatch[SortMergeJoinExecTransformer] + } + } + runQueryAndCompare( """ |select * from t1 cross join t2; diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index a3b46d7d08e1..80509e055980 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -907,9 +907,11 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::JoinRel& joinRel switch (joinRel.type()) { case ::substrait::JoinRel_JoinType_JOIN_TYPE_INNER: case ::substrait::JoinRel_JoinType_JOIN_TYPE_LEFT: + case ::substrait::JoinRel_JoinType_JOIN_TYPE_LEFT_SEMI: + case ::substrait::JoinRel_JoinType_JOIN_TYPE_RIGHT_SEMI: break; default: - LOG_VALIDATION_MSG("Sort merge join only support inner and left join."); + LOG_VALIDATION_MSG("Sort merge join only support inner, left, left semi and right semi join."); return false; } } @@ -923,7 +925,7 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::JoinRel& joinRel case ::substrait::JoinRel_JoinType_JOIN_TYPE_ANTI: break; default: - LOG_VALIDATION_MSG("Sort merge join only support inner and left join."); + LOG_VALIDATION_MSG("Join type is not supported: {}" + joinRel.type()); return false; } From 7a0af446ddcf6e78b1ad5d7e58ba45729fc3e4a1 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:38:10 +0800 Subject: [PATCH 216/402] [GLUTEN-5841][CH]Fix session timezone diff (#5892) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #5841) How was this patch tested? TEST BY UT --- .../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 6 ++++++ cpp-ch/local-engine/Common/CHUtil.cpp | 10 ++++++++++ cpp-ch/local-engine/Common/CHUtil.h | 1 + .../main/scala/org/apache/gluten/GlutenConfig.scala | 4 ++++ 4 files changed, 21 insertions(+) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index d2752a0739c0..ada980a20bc2 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -719,6 +719,12 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr "select unix_timestamp(concat(cast(l_shipdate as String), ' 00:00:00')) " + "from lineitem order by l_shipdate limit 10;")( checkGlutenOperatorMatch[ProjectExecTransformer]) + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") { + runQueryAndCompare( + "select to_unix_timestamp(concat(cast(l_shipdate as String), ' 00:00:00')) " + + "from lineitem order by l_shipdate limit 10")( + checkGlutenOperatorMatch[ProjectExecTransformer]) + } } test("test literals") { diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index e37114756f34..62b42f981168 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -657,6 +657,16 @@ void BackendInitializerUtil::initSettings(std::map & b settings.set(k, toField(k, value)); LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{} value:{}", key, value); } + else if (key == SPARK_SESSION_TIME_ZONE) + { + String time_zone_val = value; + /// Convert timezone ID like '+8:00' to GMT+8:00 + if (value.starts_with("+") || value.starts_with("-")) + time_zone_val = "GMT" + value; + time_zone_val = DateLUT::mappingForJavaTimezone(time_zone_val); + settings.set("session_timezone", time_zone_val); + LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{} value:{}", "session_timezone", time_zone_val); + } } /// Finally apply some fixed kvs to settings. diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 458eec9d3ee9..2ef3c6ef99df 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -168,6 +168,7 @@ class BackendInitializerUtil inline static const std::string SPARK_HADOOP_PREFIX = "spark.hadoop."; inline static const std::string S3A_PREFIX = "fs.s3a."; inline static const std::string SPARK_DELTA_PREFIX = "spark.databricks.delta."; + inline static const std::string SPARK_SESSION_TIME_ZONE = "spark.sql.session.timeZone"; inline static const String GLUTEN_TASK_OFFHEAP = "spark.gluten.memory.task.offHeap.size.in.bytes"; inline static const String CH_TASK_MEMORY = "off_heap_per_task"; diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 6659a42c71f9..d76e698dcf2a 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -709,6 +709,10 @@ object GlutenConfig { .filter(_._1.startsWith(SPARK_ABFS_ACCOUNT_KEY)) .foreach(entry => nativeConfMap.put(entry._1, entry._2)) + conf + .filter(_._1.startsWith(SQLConf.SESSION_LOCAL_TIMEZONE.key)) + .foreach(entry => nativeConfMap.put(entry._1, entry._2)) + // return nativeConfMap } From a5daf1c4ed59b8123838a50df22f9c744f489d11 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:41:22 +0800 Subject: [PATCH 217/402] [GLUTEN-5957][CH]Fix get_json_object on filter condition (#5989) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #5957) How was this patch tested? TEST BY UT --- .../execution/GlutenFunctionValidateSuite.scala | 6 ++++++ cpp-ch/local-engine/Parser/FilterRelParser.cpp | 3 +-- cpp-ch/local-engine/Parser/SerializedPlanParser.cpp | 3 --- cpp-ch/local-engine/Rewriter/ExpressionRewriter.h | 13 ++++++++++++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index a561fe7cb442..9327137fabe5 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -281,6 +281,12 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS } } + test("Test get_json_object 11") { + runQueryAndCompare( + "SELECT string_field1 from json_test where" + + " get_json_object(string_field1, '$.a') is not null") { _ => } + } + test("Test covar_samp") { runQueryAndCompare("SELECT covar_samp(double_field1, int_field1) from json_test") { _ => } } diff --git a/cpp-ch/local-engine/Parser/FilterRelParser.cpp b/cpp-ch/local-engine/Parser/FilterRelParser.cpp index 19facf3bff96..4c71cc3126af 100644 --- a/cpp-ch/local-engine/Parser/FilterRelParser.cpp +++ b/cpp-ch/local-engine/Parser/FilterRelParser.cpp @@ -31,7 +31,7 @@ DB::QueryPlanPtr FilterRelParser::parse(DB::QueryPlanPtr query_plan, const subst substrait::Rel final_rel = rel; rewriter.rewrite(final_rel); - const auto & filter_rel = rel.filter(); + const auto & filter_rel = final_rel.filter(); std::string filter_name; auto input_header = query_plan->getCurrentDataStream().header; @@ -66,7 +66,6 @@ DB::QueryPlanPtr FilterRelParser::parse(DB::QueryPlanPtr query_plan, const subst { steps.emplace_back(remove_null_step); } - return query_plan; } diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 25ea86e5bec0..c2bcd1a36b0a 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -1004,9 +1004,6 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( } } - if (ch_func_name == "JSON_VALUE") - result_node->function->setResolver(function_builder); - if (keep_result) actions_dag->addOrReplaceInOutputs(*result_node); diff --git a/cpp-ch/local-engine/Rewriter/ExpressionRewriter.h b/cpp-ch/local-engine/Rewriter/ExpressionRewriter.h index cdea86133eb2..c22c64eae701 100644 --- a/cpp-ch/local-engine/Rewriter/ExpressionRewriter.h +++ b/cpp-ch/local-engine/Rewriter/ExpressionRewriter.h @@ -38,7 +38,7 @@ class GetJsonObjectFunctionWriter : public RelRewriter void rewrite(substrait::Rel & rel) override { - if (!rel.has_project()) + if (!rel.has_filter() && !rel.has_project()) { return; } @@ -51,6 +51,11 @@ class GetJsonObjectFunctionWriter : public RelRewriter /// Collect all get_json_object functions and group by json strings void prepare(const substrait::Rel & rel) { + if (rel.has_filter()) + { + auto & expr = rel.filter().condition(); + prepareOnExpression(expr); + } if (rel.has_project()) { for (auto & expr : rel.project().expressions()) @@ -62,6 +67,12 @@ class GetJsonObjectFunctionWriter : public RelRewriter void rewriteImpl(substrait::Rel & rel) { + if (rel.has_filter()) + { + auto * filter = rel.mutable_filter(); + auto * expression = filter->mutable_condition(); + rewriteExpression(*expression); + } if (rel.has_project()) { auto * project = rel.mutable_project(); From 944d859004a83617a463b7e1a17bac378d5a1d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Thu, 6 Jun 2024 11:18:47 +0800 Subject: [PATCH 218/402] [GLUTEN-5787][CH]Make pipeline and shuffle exit gracefully when tasks in executors are killed or interrupted (#5839) What changes were proposed in this pull request? Changes: Clean code: remove useless JNIs and classes under cpp-ch Support cancel for all gluten processors. It was triggered when task is killed or shut down. Make sure offheap memory free does not throw exception. Ref: https://zhuanlan.zhihu.com/p/65454580 (Fixes: #5787 #5823) How was this patch tested? Manual --- .../gluten/vectorized/BatchIterator.java | 11 ++ .../CHNativeExpressionEvaluator.java | 10 +- .../clickhouse/CHIteratorApi.scala | 21 +++- .../execution/NativeFileScanColumnarRDD.scala | 12 +- .../GlutenClickHouseMetricsUTUtils.scala | 4 +- cpp-ch/local-engine/Common/BlockIterator.cpp | 6 + cpp-ch/local-engine/Common/QueryContext.cpp | 2 +- .../Operator/BlockCoalesceOperator.cpp | 53 --------- .../Operator/BlockCoalesceOperator.h | 47 -------- .../Operator/BlocksBufferPoolTransform.cpp | 2 +- .../Operator/EmptyProjectStep.cpp | 2 +- .../local-engine/Operator/ExpandTransform.cpp | 12 +- .../Operator/GraceMergingAggregatedStep.cpp | 6 +- .../PartitionColumnFillingTransform.h | 4 +- .../Operator/StreamingAggregatingStep.cpp | 7 +- .../Parser/SerializedPlanParser.cpp | 7 ++ .../Parser/SerializedPlanParser.h | 15 ++- .../Storages/SourceFromJavaIter.cpp | 15 ++- .../SubstraitSource/SubstraitFileSource.cpp | 23 +++- .../SubstraitSource/SubstraitFileSource.h | 26 +++- .../jni/ReservationListenerWrapper.cpp | 7 ++ .../jni/ReservationListenerWrapper.h | 2 + cpp-ch/local-engine/jni/jni_common.h | 27 +++++ cpp-ch/local-engine/jni/jni_error.h | 3 +- cpp-ch/local-engine/local_engine_jni.cpp | 112 ++---------------- 25 files changed, 190 insertions(+), 246 deletions(-) delete mode 100644 cpp-ch/local-engine/Operator/BlockCoalesceOperator.cpp delete mode 100644 cpp-ch/local-engine/Operator/BlockCoalesceOperator.h diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java index 5698caf02637..d674c6e90def 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java @@ -23,9 +23,11 @@ import org.apache.spark.sql.vectorized.ColumnarBatch; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; public class BatchIterator extends GeneralOutIterator { private final long handle; + private final AtomicBoolean cancelled = new AtomicBoolean(false); public BatchIterator(long handle) { super(); @@ -46,6 +48,8 @@ public String getId() { private native void nativeClose(long nativeHandle); + private native void nativeCancel(long nativeHandle); + private native IMetrics nativeFetchMetrics(long nativeHandle); @Override @@ -76,4 +80,11 @@ public IMetrics getMetricsInternal() throws IOException, ClassNotFoundException public void closeInternal() { nativeClose(handle); } + + // Used to cancel native pipeline execution when spark task is killed + public final void cancel() { + if (cancelled.compareAndSet(false, true)) { + nativeCancel(handle); + } + } } diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java index 0d307d23102d..b8b4138dc8c0 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java @@ -83,7 +83,7 @@ private Map getNativeBackendConf() { // Used by WholeStageTransform to create the native computing pipeline and // return a columnar result iterator. - public GeneralOutIterator createKernelWithBatchIterator( + public BatchIterator createKernelWithBatchIterator( byte[] wsPlan, byte[][] splitInfo, List iterList, @@ -97,11 +97,11 @@ public GeneralOutIterator createKernelWithBatchIterator( iterList.toArray(new GeneralInIterator[0]), buildNativeConf(getNativeBackendConf()), materializeInput); - return createOutIterator(handle); + return createBatchIterator(handle); } // Only for UT. - public GeneralOutIterator createKernelWithBatchIterator( + public BatchIterator createKernelWithBatchIterator( long allocId, byte[] wsPlan, byte[][] splitInfo, List iterList) { long handle = jniWrapper.nativeCreateKernelWithIterator( @@ -111,10 +111,10 @@ public GeneralOutIterator createKernelWithBatchIterator( iterList.toArray(new GeneralInIterator[0]), buildNativeConf(getNativeBackendConf()), false); - return createOutIterator(handle); + return createBatchIterator(handle); } - private GeneralOutIterator createOutIterator(long nativeHandle) { + private BatchIterator createBatchIterator(long nativeHandle) { return new BatchIterator(nativeHandle); } } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index bc16c2d77fe1..63f7eeb798f3 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -25,7 +25,7 @@ import org.apache.gluten.substrait.plan.PlanNode import org.apache.gluten.substrait.rel._ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.utils.LogLevelUtil -import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator, GeneralOutIterator} +import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator} import org.apache.spark.{InterruptibleIterator, SparkConf, TaskContext} import org.apache.spark.affinity.CHAffinity @@ -206,13 +206,19 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { val splitInfoByteArray = inputPartition .asInstanceOf[GlutenPartition] .splitInfosByteArray - val resIter: GeneralOutIterator = + val resIter = transKernel.createKernelWithBatchIterator( inputPartition.plan, splitInfoByteArray, inBatchIters, false) + context.addTaskFailureListener( + (ctx, _) => { + if (ctx.isInterrupted()) { + resIter.cancel() + } + }) context.addTaskCompletionListener[Unit](_ => resIter.close()) val iter = new Iterator[Any] { private val inputMetrics = context.taskMetrics().inputMetrics @@ -304,6 +310,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { } } var closed = false + val cancelled = false def close(): Unit = { closed = true @@ -311,6 +318,16 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { // relationHolder.clear() } + def cancel(): Unit = { + nativeIterator.cancel() + } + + context.addTaskFailureListener( + (ctx, _) => { + if (ctx.isInterrupted()) { + cancel() + } + }) context.addTaskCompletionListener[Unit](_ => close()) new CloseableCHColumnBatchIterator(resIter, Some(pipelineTime)) } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala index 624a4390d729..af512934bc96 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala @@ -17,7 +17,7 @@ package org.apache.gluten.execution import org.apache.gluten.metrics.GlutenTimeMetric -import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator, GeneralOutIterator} +import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator} import org.apache.spark.{Partition, SparkContext, SparkException, TaskContext} import org.apache.spark.rdd.RDD @@ -47,7 +47,7 @@ class NativeFileScanColumnarRDD( .asInstanceOf[GlutenPartition] .splitInfosByteArray - val resIter: GeneralOutIterator = GlutenTimeMetric.millis(scanTime) { + val resIter = GlutenTimeMetric.millis(scanTime) { _ => val transKernel = new CHNativeExpressionEvaluator() val inBatchIters = new util.ArrayList[GeneralInIterator]() @@ -58,6 +58,14 @@ class NativeFileScanColumnarRDD( false ) } + TaskContext + .get() + .addTaskFailureListener( + (ctx, _) => { + if (ctx.isInterrupted()) { + resIter.cancel() + } + }) TaskContext.get().addTaskCompletionListener[Unit](_ => resIter.close()) val iter: Iterator[ColumnarBatch] = new Iterator[ColumnarBatch] { var scanTotalTime = 0L diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala index bc395ca88e9a..ee0ad8039afc 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala @@ -20,7 +20,7 @@ import org.apache.gluten.execution.WholeStageTransformer import org.apache.gluten.memory.alloc.CHNativeMemoryAllocators import org.apache.gluten.metrics.{MetricsUtil, NativeMetrics} import org.apache.gluten.utils.SubstraitPlanPrinterUtil -import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, GeneralInIterator, GeneralOutIterator} +import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, GeneralInIterator} import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.catalyst.expressions.Attribute @@ -47,7 +47,7 @@ object GlutenClickHouseMetricsUTUtils { val transKernel = new CHNativeExpressionEvaluator() val mockMemoryAllocator = CHNativeMemoryAllocators.contextInstanceForUT() - val resIter: GeneralOutIterator = transKernel.createKernelWithBatchIterator( + val resIter = transKernel.createKernelWithBatchIterator( mockMemoryAllocator.getNativeInstanceId, substraitPlan.toByteArray, new Array[Array[Byte]](0), diff --git a/cpp-ch/local-engine/Common/BlockIterator.cpp b/cpp-ch/local-engine/Common/BlockIterator.cpp index 1a76f646b20a..464701893207 100644 --- a/cpp-ch/local-engine/Common/BlockIterator.cpp +++ b/cpp-ch/local-engine/Common/BlockIterator.cpp @@ -34,24 +34,30 @@ void local_engine::BlockIterator::checkNextValid() throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Block iterator next should after hasNext"); } } + void BlockIterator::produce() { consumed = false; } + void BlockIterator::consume() { consumed = true; } + bool BlockIterator::isConsumed() const { return consumed; } + DB::Block & BlockIterator::currentBlock() { return cached_block; } + void BlockIterator::setCurrentBlock(DB::Block & block) { cached_block = block; } + } diff --git a/cpp-ch/local-engine/Common/QueryContext.cpp b/cpp-ch/local-engine/Common/QueryContext.cpp index c659e6f34ea1..f4d39c612430 100644 --- a/cpp-ch/local-engine/Common/QueryContext.cpp +++ b/cpp-ch/local-engine/Common/QueryContext.cpp @@ -67,7 +67,7 @@ int64_t initializeQuery(ReservationListenerWrapperPtr listener) else listener->reserve(size); }; - CurrentMemoryTracker::before_free = [listener](Int64 size) -> void { listener->free(size); }; + CurrentMemoryTracker::before_free = [listener](Int64 size) -> void { listener->tryFree(size); }; CurrentMemoryTracker::current_memory = [listener]() -> Int64 { return listener->currentMemory(); }; allocator_map.insert(allocator_id, allocator_context); return allocator_id; diff --git a/cpp-ch/local-engine/Operator/BlockCoalesceOperator.cpp b/cpp-ch/local-engine/Operator/BlockCoalesceOperator.cpp deleted file mode 100644 index 756249e8a571..000000000000 --- a/cpp-ch/local-engine/Operator/BlockCoalesceOperator.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "BlockCoalesceOperator.h" -#include - -namespace local_engine -{ - -void BlockCoalesceOperator::mergeBlock(DB::Block & block) -{ - block_buffer.add(block, 0, static_cast(block.rows())); -} - -bool BlockCoalesceOperator::isFull() -{ - return block_buffer.size() >= buf_size; -} - -DB::Block * BlockCoalesceOperator::releaseBlock() -{ - clearCache(); - cached_block = new DB::Block(block_buffer.releaseColumns()); - return cached_block; -} - -BlockCoalesceOperator::~BlockCoalesceOperator() -{ - clearCache(); -} - -void BlockCoalesceOperator::clearCache() -{ - if (cached_block) - { - delete cached_block; - cached_block = nullptr; - } -} -} diff --git a/cpp-ch/local-engine/Operator/BlockCoalesceOperator.h b/cpp-ch/local-engine/Operator/BlockCoalesceOperator.h deleted file mode 100644 index 2b67b40cea1b..000000000000 --- a/cpp-ch/local-engine/Operator/BlockCoalesceOperator.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include - -namespace DB -{ -class Block; -} - -namespace local_engine -{ - -class BlockCoalesceOperator -{ -public: - explicit BlockCoalesceOperator(size_t buf_size_) : buf_size(buf_size_) { } - ~BlockCoalesceOperator(); - - void mergeBlock(DB::Block & block); - bool isFull(); - DB::Block * releaseBlock(); - -private: - void clearCache(); - - size_t buf_size; - ColumnsBuffer block_buffer; - DB::Block * cached_block = nullptr; - -}; -} diff --git a/cpp-ch/local-engine/Operator/BlocksBufferPoolTransform.cpp b/cpp-ch/local-engine/Operator/BlocksBufferPoolTransform.cpp index 3427a81c60c3..16a5bd5d2619 100644 --- a/cpp-ch/local-engine/Operator/BlocksBufferPoolTransform.cpp +++ b/cpp-ch/local-engine/Operator/BlocksBufferPoolTransform.cpp @@ -43,7 +43,7 @@ DB::IProcessor::Status BlocksBufferPoolTransform::prepare() { auto & output = outputs.front(); auto & input = inputs.front(); - if (output.isFinished()) + if (output.isFinished() || isCancelled()) { input.close(); return Status::Finished; diff --git a/cpp-ch/local-engine/Operator/EmptyProjectStep.cpp b/cpp-ch/local-engine/Operator/EmptyProjectStep.cpp index 58cb33e59a6f..62991585f3b5 100644 --- a/cpp-ch/local-engine/Operator/EmptyProjectStep.cpp +++ b/cpp-ch/local-engine/Operator/EmptyProjectStep.cpp @@ -39,7 +39,7 @@ class EmptyProject : public DB::IProcessor { auto & output = outputs.front(); auto & input = inputs.front(); - if (output.isFinished()) + if (output.isFinished() || isCancelled()) { input.close(); return Status::Finished; diff --git a/cpp-ch/local-engine/Operator/ExpandTransform.cpp b/cpp-ch/local-engine/Operator/ExpandTransform.cpp index d48d484397c2..106c38e2d8c3 100644 --- a/cpp-ch/local-engine/Operator/ExpandTransform.cpp +++ b/cpp-ch/local-engine/Operator/ExpandTransform.cpp @@ -48,7 +48,7 @@ ExpandTransform::Status ExpandTransform::prepare() auto & output = outputs.front(); auto & input = inputs.front(); - if (output.isFinished()) + if (output.isFinished() || isCancelled()) { input.close(); return Status::Finished; @@ -79,12 +79,12 @@ ExpandTransform::Status ExpandTransform::prepare() if (!input.hasData()) return Status::NeedData; - + input_chunk = input.pull(true); has_input = true; expand_expr_iterator = 0; } - + return Status::Ready; } @@ -92,6 +92,7 @@ void ExpandTransform::work() { if (expand_expr_iterator >= project_set_exprs.getExpandRows()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "expand_expr_iterator >= project_set_exprs.getExpandRows()"); + const auto & original_cols = input_chunk.getColumns(); size_t rows = input_chunk.getNumRows(); DB::Columns cols; @@ -139,8 +140,9 @@ void ExpandTransform::work() } } output_chunk = DB::Chunk(cols, rows); - expand_expr_iterator += 1; - has_output = expand_expr_iterator <= project_set_exprs.getExpandRows(); + has_output = true; + + ++expand_expr_iterator; has_input = expand_expr_iterator < project_set_exprs.getExpandRows(); } } diff --git a/cpp-ch/local-engine/Operator/GraceMergingAggregatedStep.cpp b/cpp-ch/local-engine/Operator/GraceMergingAggregatedStep.cpp index ca86785febfe..a9a2df276a59 100644 --- a/cpp-ch/local-engine/Operator/GraceMergingAggregatedStep.cpp +++ b/cpp-ch/local-engine/Operator/GraceMergingAggregatedStep.cpp @@ -146,7 +146,7 @@ GraceMergingAggregatedTransform::Status GraceMergingAggregatedTransform::prepare { auto & output = outputs.front(); auto & input = inputs.front(); - if (output.isFinished()) + if (output.isFinished() || isCancelled()) { input.close(); return Status::Finished; @@ -224,7 +224,7 @@ void GraceMergingAggregatedTransform::work() block_converter = prepareBucketOutputBlocks(current_bucket_index); if (block_converter) break; - current_bucket_index++; + current_bucket_index++; } } if (!block_converter) @@ -455,7 +455,7 @@ std::unique_ptr GraceMergingAggregatedTransform::pr block = {}; } } - + if (buffer_file_stream.original_file_stream) { buffer_file_stream.original_file_stream->finishWriting(); diff --git a/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.h b/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.h index 2eac10f3975b..692991b3fc9d 100644 --- a/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.h +++ b/cpp-ch/local-engine/Operator/PartitionColumnFillingTransform.h @@ -25,9 +25,11 @@ class PartitionColumnFillingTransform : public DB::ISimpleTransform public: PartitionColumnFillingTransform( const DB::Block & input_, const DB::Block & output_, const String & partition_col_name_, const String & partition_col_value_); - void transform(DB::Chunk & chunk) override; + String getName() const override { return "PartitionColumnFillingTransform"; } + void transform(DB::Chunk & chunk) override; + private: DB::ColumnPtr createPartitionColumn(); diff --git a/cpp-ch/local-engine/Operator/StreamingAggregatingStep.cpp b/cpp-ch/local-engine/Operator/StreamingAggregatingStep.cpp index 698d353b1d4b..65d77f8e968f 100644 --- a/cpp-ch/local-engine/Operator/StreamingAggregatingStep.cpp +++ b/cpp-ch/local-engine/Operator/StreamingAggregatingStep.cpp @@ -67,11 +67,12 @@ StreamingAggregatingTransform::Status StreamingAggregatingTransform::prepare() { auto & output = outputs.front(); auto & input = inputs.front(); - if (output.isFinished()) + if (output.isFinished() || isCancelled()) { input.close(); return Status::Finished; } + if (has_output) { if (output.canPush()) @@ -140,10 +141,10 @@ bool StreamingAggregatingTransform::needEvict() auto max_mem_used = static_cast(context->getSettingsRef().max_memory_usage * max_allowed_memory_usage_ratio); auto current_result_rows = data_variants->size(); - /// avoid evict empty or too small aggregated results. + /// avoid evict empty or too small aggregated results. if (current_result_rows < aggregated_keys_before_evict) return false; - + /// If the grouping keys is high cardinality, we should evict data variants early, and avoid to use a big /// hash table. if (static_cast(total_output_rows)/total_input_rows > high_cardinality_threshold) diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index c2bcd1a36b0a..1c907035a5ce 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -2043,6 +2043,7 @@ LocalExecutor::~LocalExecutor() { if (context->getConfigRef().getBool("dump_pipeline", false)) LOG_INFO(&Poco::Logger::get("LocalExecutor"), "Dump pipeline:\n{}", dumpPipeline()); + if (spark_buffer) { ch_column_to_spark_row->freeMem(spark_buffer->address, spark_buffer->size); @@ -2166,6 +2167,12 @@ Block * LocalExecutor::nextColumnar() return columnar_batch; } +void LocalExecutor::cancel() +{ + if (executor) + executor->cancel(); +} + Block & LocalExecutor::getHeader() { return header; diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 73448b0690c2..d0a16ec71a47 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -402,21 +402,26 @@ class LocalExecutor : public BlockIterator public: LocalExecutor() = default; explicit LocalExecutor(ContextPtr context); + ~LocalExecutor(); + void execute(QueryPlanPtr query_plan); SparkRowInfoPtr next(); Block * nextColumnar(); bool hasNext(); - ~LocalExecutor(); - Block & getHeader(); + /// Stop execution, used when task receives shutdown command or executor receives SIGTERM signal + void cancel(); + Block & getHeader(); RelMetricPtr getMetric() const { return metric; } void setMetric(RelMetricPtr metric_) { metric = metric_; } - void setExtraPlanHolder(std::vector & extra_plan_holder_) { extra_plan_holder = std::move(extra_plan_holder_); } - private: std::unique_ptr writeBlockToSparkRow(DB::Block & block); + + /// Dump processor runtime information to log + std::string dumpPipeline(); + QueryPipeline query_pipeline; std::unique_ptr executor; Block header; @@ -427,8 +432,6 @@ class LocalExecutor : public BlockIterator RelMetricPtr metric; std::vector extra_plan_holder; - /// Dump processor runtime information to log - std::string dumpPipeline(); }; diff --git a/cpp-ch/local-engine/Storages/SourceFromJavaIter.cpp b/cpp-ch/local-engine/Storages/SourceFromJavaIter.cpp index 54d1d253e539..37501e98504a 100644 --- a/cpp-ch/local-engine/Storages/SourceFromJavaIter.cpp +++ b/cpp-ch/local-engine/Storages/SourceFromJavaIter.cpp @@ -53,12 +53,11 @@ static DB::Block getRealHeader(const DB::Block & header) DB::Block * SourceFromJavaIter::peekBlock(JNIEnv * env, jobject java_iter) { jboolean has_next = safeCallBooleanMethod(env, java_iter, serialized_record_batch_iterator_hasNext); - if (has_next) - { - jbyteArray block = static_cast(safeCallObjectMethod(env, java_iter, serialized_record_batch_iterator_next)); - return reinterpret_cast(byteArrayToLong(env, block)); - } - return nullptr; + if (!has_next) + return nullptr; + + jbyteArray block = static_cast(safeCallObjectMethod(env, java_iter, serialized_record_batch_iterator_next)); + return reinterpret_cast(byteArrayToLong(env, block)); } @@ -75,6 +74,9 @@ SourceFromJavaIter::SourceFromJavaIter( DB::Chunk SourceFromJavaIter::generate() { + if (isCancelled()) + return {}; + GET_JNIENV(env) SCOPE_EXIT({CLEAN_JNIENV}); @@ -152,6 +154,7 @@ void SourceFromJavaIter::convertNullable(DB::Chunk & chunk) chunk.setColumns(columns, rows); } + DB::ColumnPtr SourceFromJavaIter::convertNestedNullable(const DB::ColumnPtr & column, const DB::DataTypePtr & target_type) { DB::WhichDataType column_type(column->getDataType()); diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp index 80dccf759060..5b872244eab5 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp @@ -104,6 +104,9 @@ DB::Chunk SubstraitFileSource::generate() bool SubstraitFileSource::tryPrepareReader() { + if (isCancelled()) + return false; + if (file_reader) return true; @@ -140,6 +143,13 @@ bool SubstraitFileSource::tryPrepareReader() return true; } + +void SubstraitFileSource::onCancel() +{ + if (file_reader) + file_reader->cancel(); +} + DB::ColumnPtr FileReaderWrapper::createConstColumn(DB::DataTypePtr data_type, const DB::Field & field, size_t rows) { auto nested_type = DB::removeNullable(data_type); @@ -280,9 +290,13 @@ ConstColumnsFileReader::ConstColumnsFileReader(FormatFilePtr file_, DB::ContextP remained_rows = *rows; } + bool ConstColumnsFileReader::pull(DB::Chunk & chunk) { - if (!remained_rows) [[unlikely]] + if (isCancelled()) + return false; + + if (!remained_rows) return false; size_t to_read_rows = 0; @@ -296,6 +310,7 @@ bool ConstColumnsFileReader::pull(DB::Chunk & chunk) to_read_rows = block_size; remained_rows -= block_size; } + DB::Columns res_columns; if (const size_t col_num = header.columns()) { @@ -307,8 +322,9 @@ bool ConstColumnsFileReader::pull(DB::Chunk & chunk) auto type = col_with_name_and_type.type; const auto & name = col_with_name_and_type.name; auto it = partition_values.find(name); - if (it == partition_values.end()) [[unlikely]] + if (it == partition_values.end()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknow partition column : {}", name); + res_columns.emplace_back(createColumn(it->second, type, to_read_rows)); } } @@ -331,6 +347,9 @@ NormalFileReader::NormalFileReader( bool NormalFileReader::pull(DB::Chunk & chunk) { + if (isCancelled()) + return false; + DB::Chunk raw_chunk = input_format->input->generate(); const size_t rows = raw_chunk.getNumRows(); if (!rows) diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h index 973f3cd35b2a..650ec5d967a0 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h +++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h @@ -34,6 +34,15 @@ class FileReaderWrapper virtual ~FileReaderWrapper() = default; virtual bool pull(DB::Chunk & chunk) = 0; + void cancel() + { + bool already_cancelled = is_cancelled.exchange(true, std::memory_order_acq_rel); + if (!already_cancelled) + onCancel(); + } + + bool isCancelled() const { return is_cancelled.load(std::memory_order_acquire); } + /// Apply key condition to the reader, if use_local_format is true, column_index_filter will be used /// otherwise it will be ignored virtual void applyKeyCondition( @@ -42,7 +51,11 @@ class FileReaderWrapper } protected: + virtual void onCancel() {}; + FormatFilePtr file; + std::atomic is_cancelled{false}; + static DB::ColumnPtr createConstColumn(DB::DataTypePtr type, const DB::Field & field, size_t rows); static DB::ColumnPtr createColumn(const String & value, DB::DataTypePtr type, size_t rows); @@ -68,10 +81,14 @@ class NormalFileReader : public FileReaderWrapper } private: + void onCancel() override + { + input_format->input->cancel(); + } + DB::ContextPtr context; DB::Block to_read_header; DB::Block output_header; - FormatFile::InputFormatPtr input_format; }; @@ -89,6 +106,7 @@ class ConstColumnsFileReader : public FileReaderWrapper ConstColumnsFileReader( FormatFilePtr file_, DB::ContextPtr context_, const DB::Block & header_, size_t block_size_ = DB::DEFAULT_BLOCK_SIZE); ~ConstColumnsFileReader() override = default; + bool pull(DB::Chunk & chunk) override; private: @@ -112,6 +130,9 @@ class SubstraitFileSource : public DB::SourceWithKeyCondition DB::Chunk generate() override; private: + bool tryPrepareReader(); + void onCancel() override; + DB::ContextPtr context; DB::Block output_header; /// Sample header may contains partitions keys DB::Block to_read_header; // Sample header not include partition keys @@ -120,9 +141,6 @@ class SubstraitFileSource : public DB::SourceWithKeyCondition UInt32 current_file_index = 0; std::unique_ptr file_reader; ReadBufferBuilderPtr read_buffer_builder; - ColumnIndexFilterPtr column_index_filter; - - bool tryPrepareReader(); }; } diff --git a/cpp-ch/local-engine/jni/ReservationListenerWrapper.cpp b/cpp-ch/local-engine/jni/ReservationListenerWrapper.cpp index 65b29c2a2d1c..dad0ecf66ed7 100644 --- a/cpp-ch/local-engine/jni/ReservationListenerWrapper.cpp +++ b/cpp-ch/local-engine/jni/ReservationListenerWrapper.cpp @@ -58,6 +58,13 @@ void ReservationListenerWrapper::free(int64_t size) CLEAN_JNIENV } +void ReservationListenerWrapper::tryFree(int64_t size) +{ + GET_JNIENV(env) + tryCallVoidMethod(env, listener, reservation_listener_unreserve, size); + CLEAN_JNIENV +} + size_t ReservationListenerWrapper::currentMemory() { GET_JNIENV(env) diff --git a/cpp-ch/local-engine/jni/ReservationListenerWrapper.h b/cpp-ch/local-engine/jni/ReservationListenerWrapper.h index 1dfb3671f21b..a4d26cb5417e 100644 --- a/cpp-ch/local-engine/jni/ReservationListenerWrapper.h +++ b/cpp-ch/local-engine/jni/ReservationListenerWrapper.h @@ -35,6 +35,8 @@ class ReservationListenerWrapper void reserve(int64_t size); void reserveOrThrow(int64_t size); void free(int64_t size); + /// Make sure destructors in CH Backend do not throw exceptions + void tryFree(int64_t size); size_t currentMemory(); diff --git a/cpp-ch/local-engine/jni/jni_common.h b/cpp-ch/local-engine/jni/jni_common.h index c1cc805aa3ed..8d14370835c4 100644 --- a/cpp-ch/local-engine/jni/jni_common.h +++ b/cpp-ch/local-engine/jni/jni_common.h @@ -28,6 +28,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int UNKNOWN_TYPE; } } @@ -62,6 +63,24 @@ jbyteArray stringTojbyteArray(JNIEnv * env, const std::string & str); throw DB::Exception::createRuntime(DB::ErrorCodes::LOGICAL_ERROR, msg); \ } +#define TRY_LOCAL_ENGINE_JNI_JMETHOD_START +#define TRY_LOCAL_ENGINE_JNI_JMETHOD_END(env) \ + if ((env)->ExceptionCheck()) \ + { \ + LOG_ERROR(&Poco::Logger::get("local_engine"), "Enter java exception handle."); \ + auto excp = (env)->ExceptionOccurred(); \ + (env)->ExceptionDescribe(); \ + (env)->ExceptionClear(); \ + jclass cls = (env)->GetObjectClass(excp); \ + jmethodID mid = env->GetMethodID(cls, "toString", "()Ljava/lang/String;"); \ + jstring jmsg = static_cast((env)->CallObjectMethod(excp, mid)); \ + const char * nmsg = (env)->GetStringUTFChars(jmsg, NULL); \ + std::string msg = std::string(nmsg); \ + env->ReleaseStringUTFChars(jmsg, nmsg); \ + LOG_WARNING(&Poco::Logger::get("local_engine"), "Ignore java exception: {}", msg); \ + } + + template jobject safeCallObjectMethod(JNIEnv * env, jobject obj, jmethodID method_id, Args... args) { @@ -106,6 +125,14 @@ void safeCallVoidMethod(JNIEnv * env, jobject obj, jmethodID method_id, Args... LOCAL_ENGINE_JNI_JMETHOD_END(env) } +template +void tryCallVoidMethod(JNIEnv * env, jobject obj, jmethodID method_id, Args... args) +{ + TRY_LOCAL_ENGINE_JNI_JMETHOD_START + env->CallVoidMethod(obj, method_id, args...); + TRY_LOCAL_ENGINE_JNI_JMETHOD_END(env) +} + template jlong safeCallStaticLongMethod(JNIEnv * env, jclass clazz, jmethodID method_id, Args... args) { diff --git a/cpp-ch/local-engine/jni/jni_error.h b/cpp-ch/local-engine/jni/jni_error.h index 216a5da9392f..c6f46bc8fc6e 100644 --- a/cpp-ch/local-engine/jni/jni_error.h +++ b/cpp-ch/local-engine/jni/jni_error.h @@ -37,6 +37,8 @@ class JniErrorsGlobalState : boost::noncopyable ~JniErrorsGlobalState() = default; static JniErrorsGlobalState & instance(); + static void throwException(JNIEnv * env, jclass exception_class, const std::string & message, const std::string & stack_trace = ""); + void initialize(JNIEnv * env_); void destroy(JNIEnv * env); @@ -48,7 +50,6 @@ class JniErrorsGlobalState : boost::noncopyable void throwException(JNIEnv * env, const DB::Exception & e); void throwException(JNIEnv * env, const std::exception & e); - static void throwException(JNIEnv * env, jclass exception_class, const std::string & message, const std::string & stack_trace = ""); void throwRuntimeException(JNIEnv * env, const std::string & message, const std::string & stack_trace = ""); diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 1c088720dbcb..be28b9fabeff 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -285,6 +284,7 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ plan_string.assign(reinterpret_cast(plan_address), plan_size); auto query_plan = parser.parse(plan_string); local_engine::LocalExecutor * executor = new local_engine::LocalExecutor(query_context); + LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); executor->setMetric(parser.getMetric()); executor->setExtraPlanHolder(parser.extra_plan_holder); executor->execute(std::move(query_plan)); @@ -294,44 +294,6 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ LOCAL_ENGINE_JNI_METHOD_END(env, -1) } -JNIEXPORT jboolean Java_org_apache_gluten_row_RowIterator_nativeHasNext(JNIEnv * env, jobject /*obj*/, jlong executor_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); - return executor->hasNext(); - LOCAL_ENGINE_JNI_METHOD_END(env, false) -} - -JNIEXPORT jobject Java_org_apache_gluten_row_RowIterator_nativeNext(JNIEnv * env, jobject /*obj*/, jlong executor_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); - local_engine::SparkRowInfoPtr spark_row_info = executor->next(); - - auto * offsets_arr = env->NewLongArray(spark_row_info->getNumRows()); - const auto * offsets_src = reinterpret_cast(spark_row_info->getOffsets().data()); - env->SetLongArrayRegion(offsets_arr, 0, spark_row_info->getNumRows(), offsets_src); - auto * lengths_arr = env->NewLongArray(spark_row_info->getNumRows()); - const auto * lengths_src = reinterpret_cast(spark_row_info->getLengths().data()); - env->SetLongArrayRegion(lengths_arr, 0, spark_row_info->getNumRows(), lengths_src); - int64_t address = reinterpret_cast(spark_row_info->getBufferAddress()); - int64_t column_number = reinterpret_cast(spark_row_info->getNumCols()); - int64_t total_size = reinterpret_cast(spark_row_info->getTotalBytes()); - - jobject spark_row_info_object - = env->NewObject(spark_row_info_class, spark_row_info_constructor, offsets_arr, lengths_arr, address, column_number, total_size); - return spark_row_info_object; - LOCAL_ENGINE_JNI_METHOD_END(env, nullptr) -} - -JNIEXPORT void Java_org_apache_gluten_row_RowIterator_nativeClose(JNIEnv * env, jobject /*obj*/, jlong executor_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); - delete executor; - LOCAL_ENGINE_JNI_METHOD_END(env, ) -} - // Columnar Iterator JNIEXPORT jboolean Java_org_apache_gluten_vectorized_BatchIterator_nativeHasNext(JNIEnv * env, jobject /*obj*/, jlong executor_address) { @@ -346,15 +308,24 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_BatchIterator_nativeCHNext(JNI LOCAL_ENGINE_JNI_METHOD_START local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); DB::Block * column_batch = executor->nextColumnar(); - // LOG_DEBUG(&Poco::Logger::get("jni"), "row size of the column batch: {}", column_batch->rows()); return reinterpret_cast(column_batch); LOCAL_ENGINE_JNI_METHOD_END(env, -1) } +JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeCancel(JNIEnv * env, jobject /*obj*/, jlong executor_address) +{ + LOCAL_ENGINE_JNI_METHOD_START + local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); + executor->cancel(); + LOG_INFO(&Poco::Logger::get("jni"), "Cancel LocalExecutor {}", reinterpret_cast(executor)); + LOCAL_ENGINE_JNI_METHOD_END(env, ) +} + JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeClose(JNIEnv * env, jobject /*obj*/, jlong executor_address) { LOCAL_ENGINE_JNI_METHOD_START local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); + LOG_INFO(&Poco::Logger::get("jni"), "Finalize LocalExecutor {}", reinterpret_cast(executor)); delete executor; LOCAL_ENGINE_JNI_METHOD_END(env, ) } @@ -372,21 +343,6 @@ JNIEXPORT jobject Java_org_apache_gluten_vectorized_BatchIterator_nativeFetchMet LOCAL_ENGINE_JNI_METHOD_END(env, nullptr) } -JNIEXPORT void -Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeSetJavaTmpDir(JNIEnv * /*env*/, jobject /*obj*/, jstring /*dir*/) -{ -} - -JNIEXPORT void -Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeSetBatchSize(JNIEnv * /*env*/, jobject /*obj*/, jint /*batch_size*/) -{ -} - -JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeSetMetricsTime( - JNIEnv * /*env*/, jobject /*obj*/, jboolean /*setMetricsTime*/) -{ -} - JNIEXPORT jboolean Java_org_apache_gluten_vectorized_CHColumnVector_nativeHasNull(JNIEnv * env, jobject obj, jlong block_address, jint column_position) { @@ -603,52 +559,6 @@ JNIEXPORT void Java_org_apache_gluten_vectorized_CHStreamReader_nativeClose(JNIE LOCAL_ENGINE_JNI_METHOD_END(env, ) } -JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHCoalesceOperator_createNativeOperator(JNIEnv * env, jobject /*obj*/, jint buf_size) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::BlockCoalesceOperator * instance = new local_engine::BlockCoalesceOperator(buf_size); - return reinterpret_cast(instance); - LOCAL_ENGINE_JNI_METHOD_END(env, -1) -} - -JNIEXPORT void Java_org_apache_gluten_vectorized_CHCoalesceOperator_nativeMergeBlock( - JNIEnv * env, jobject /*obj*/, jlong instance_address, jlong block_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::BlockCoalesceOperator * instance = reinterpret_cast(instance_address); - DB::Block * block = reinterpret_cast(block_address); - auto new_block = DB::Block(*block); - instance->mergeBlock(new_block); - LOCAL_ENGINE_JNI_METHOD_END(env, ) -} - -JNIEXPORT jboolean Java_org_apache_gluten_vectorized_CHCoalesceOperator_nativeIsFull(JNIEnv * env, jobject /*obj*/, jlong instance_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::BlockCoalesceOperator * instance = reinterpret_cast(instance_address); - bool full = instance->isFull(); - return full ? JNI_TRUE : JNI_FALSE; - LOCAL_ENGINE_JNI_METHOD_END(env, false) -} - -JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHCoalesceOperator_nativeRelease(JNIEnv * env, jobject /*obj*/, jlong instance_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::BlockCoalesceOperator * instance = reinterpret_cast(instance_address); - auto * block = instance->releaseBlock(); - Int64 address = reinterpret_cast(block); - return address; - LOCAL_ENGINE_JNI_METHOD_END(env, -1) -} - -JNIEXPORT void Java_org_apache_gluten_vectorized_CHCoalesceOperator_nativeClose(JNIEnv * env, jobject /*obj*/, jlong instance_address) -{ - LOCAL_ENGINE_JNI_METHOD_START - local_engine::BlockCoalesceOperator * instance = reinterpret_cast(instance_address); - delete instance; - LOCAL_ENGINE_JNI_METHOD_END(env, ) -} - // Splitter Jni Wrapper JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_nativeMake( JNIEnv * env, From 95dcdbdbd9f4f17de40884b74a65564bba2a9bf9 Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Thu, 6 Jun 2024 13:01:25 +0800 Subject: [PATCH 219/402] [CORE] Rename CoalesceExecTransformer to ColumnarCoalesceExec (#6000) --- ...Transformer.scala => ColumnarCoalesceExec.scala} | 13 +++++-------- .../extension/columnar/OffloadSingleNode.scala | 2 +- .../extension/columnar/TransformHintRule.scala | 5 +++-- 3 files changed, 9 insertions(+), 11 deletions(-) rename gluten-core/src/main/scala/org/apache/gluten/execution/{CoalesceExecTransformer.scala => ColumnarCoalesceExec.scala} (87%) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/CoalesceExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala similarity index 87% rename from gluten-core/src/main/scala/org/apache/gluten/execution/CoalesceExecTransformer.scala rename to gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala index 8f30805beeed..f40a7f8f07f6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/CoalesceExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/ColumnarCoalesceExec.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.extension.{GlutenPlan, ValidationResult} +import org.apache.gluten.extension.GlutenPlan import org.apache.spark.{Partition, SparkContext, TaskContext} import org.apache.spark.rdd.RDD @@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, SinglePartiti import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} import org.apache.spark.sql.vectorized.ColumnarBatch -case class CoalesceExecTransformer(numPartitions: Int, child: SparkPlan) +case class ColumnarCoalesceExec(numPartitions: Int, child: SparkPlan) extends UnaryExecNode with GlutenPlan { @@ -38,9 +38,6 @@ case class CoalesceExecTransformer(numPartitions: Int, child: SparkPlan) if (numPartitions == 1) SinglePartition else UnknownPartitioning(numPartitions) } - override protected def doValidateInternal(): ValidationResult = - ValidationResult.ok - override protected def doExecute(): RDD[InternalRow] = { throw new UnsupportedOperationException() } @@ -49,18 +46,18 @@ case class CoalesceExecTransformer(numPartitions: Int, child: SparkPlan) if (numPartitions == 1 && child.executeColumnar().getNumPartitions < 1) { // Make sure we don't output an RDD with 0 partitions, when claiming that we have a // `SinglePartition`. - new CoalesceExecTransformer.EmptyRDDWithPartitions(sparkContext, numPartitions) + new ColumnarCoalesceExec.EmptyRDDWithPartitions(sparkContext, numPartitions) } else { child.executeColumnar().coalesce(numPartitions, shuffle = false) } } - override protected def withNewChildInternal(newChild: SparkPlan): CoalesceExecTransformer = + override protected def withNewChildInternal(newChild: SparkPlan): ColumnarCoalesceExec = copy(child = newChild) } -object CoalesceExecTransformer { +object ColumnarCoalesceExec { class EmptyRDDWithPartitions(@transient private val sc: SparkContext, numPartitions: Int) extends RDD[ColumnarBatch](sc, Nil) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index b82254072033..39cc8ad2e2e6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -281,7 +281,7 @@ object OffloadOthers { applyScanTransformer(plan) case plan: CoalesceExec => logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - CoalesceExecTransformer(plan.numPartitions, plan.child) + ColumnarCoalesceExec(plan.numPartitions, plan.child) case plan: ProjectExec => val columnarChild = plan.child logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index 1da0c0db8a6d..ca35c74f6892 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -487,8 +487,9 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { ) transformer.doValidate().tagOnFallback(plan) case plan: CoalesceExec => - val transformer = CoalesceExecTransformer(plan.numPartitions, plan.child) - transformer.doValidate().tagOnFallback(plan) + ColumnarCoalesceExec(plan.numPartitions, plan.child) + .doValidate() + .tagOnFallback(plan) case plan: GlobalLimitExec => val (limit, offset) = SparkShimLoader.getSparkShims.getLimitAndOffsetFromGlobalLimit(plan) From a2e96f7035f4943a6f7a54c272c99693dfebcd1e Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:15:40 +0800 Subject: [PATCH 220/402] [VL] Handle try_subtract, try_multiply, try_divide (#5985) [VL] Handle try_subtract, try_multiply, try_divide. --- .../velox/VeloxSparkPlanExecApi.scala | 28 +++++---- .../ScalarFunctionsValidateSuite.scala | 24 ++++++++ .../functions/RegistrationAllFunctions.cc | 5 ++ cpp/velox/substrait/SubstraitParser.cc | 1 - docs/velox-backend-support-progress.md | 2 +- .../gluten/backendsapi/SparkPlanExecApi.scala | 12 ++-- .../expression/ExpressionConverter.scala | 58 +++++++++++++++++-- .../utils/velox/VeloxTestSettings.scala | 3 +- .../expressions/GlutenTryEvalSuite.scala | 21 +++++++ .../utils/velox/VeloxTestSettings.scala | 3 +- .../expressions/GlutenTryEvalSuite.scala | 21 +++++++ .../gluten/expression/ExpressionNames.scala | 5 +- .../sql/shims/spark34/Spark34Shims.scala | 6 ++ .../sql/shims/spark35/Spark35Shims.scala | 6 ++ 14 files changed, 171 insertions(+), 24 deletions(-) create mode 100644 gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala create mode 100644 gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 16c11f111abc..f8af80a9b44d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -123,42 +123,50 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { original) } - override def genTryAddTransformer( + override def genTryArithmeticTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, - original: TryEval): ExpressionTransformer = { + original: TryEval, + checkArithmeticExprName: String): ExpressionTransformer = { if (SparkShimLoader.getSparkShims.withAnsiEvalMode(original.child)) { - throw new GlutenNotSupportException(s"add with ansi mode is not supported") + throw new GlutenNotSupportException( + s"${original.child.prettyName} with ansi mode is not supported") } original.child.dataType match { case LongType | IntegerType | ShortType | ByteType => - case _ => throw new GlutenNotSupportException(s"try_add is not supported") + case _ => throw new GlutenNotSupportException(s"$substraitExprName is not supported") } // Offload to velox for only IntegralTypes. GenericExpressionTransformer( substraitExprName, - Seq(GenericExpressionTransformer(ExpressionNames.TRY_ADD, Seq(left, right), original)), + Seq(GenericExpressionTransformer(checkArithmeticExprName, Seq(left, right), original)), original) } - override def genAddTransformer( + /** + * Map arithmetic expr to different functions: substraitExprName or try(checkArithmeticExprName) + * based on EvalMode. + */ + override def genArithmeticTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, - original: Add): ExpressionTransformer = { + original: Expression, + checkArithmeticExprName: String): ExpressionTransformer = { if (SparkShimLoader.getSparkShims.withTryEvalMode(original)) { original.dataType match { case LongType | IntegerType | ShortType | ByteType => - case _ => throw new GlutenNotSupportException(s"try_add is not supported") + case _ => + throw new GlutenNotSupportException(s"$substraitExprName with try mode is not supported") } // Offload to velox for only IntegralTypes. GenericExpressionTransformer( ExpressionMappings.expressionsMap(classOf[TryEval]), - Seq(GenericExpressionTransformer(ExpressionNames.TRY_ADD, Seq(left, right), original)), + Seq(GenericExpressionTransformer(checkArithmeticExprName, Seq(left, right), original)), original) } else if (SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) { - throw new GlutenNotSupportException(s"add with ansi mode is not supported") + throw new GlutenNotSupportException(s"$substraitExprName with ansi mode is not supported") } else { GenericExpressionTransformer(substraitExprName, Seq(left, right), original) } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 8802c61c5f04..6df3a062331f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -843,6 +843,30 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + testWithSpecifiedSparkVersion("try_subtract", Some("3.3")) { + runQueryAndCompare( + "select try_subtract(2147483647, cast(l_orderkey as int)), " + + "try_subtract(-2147483648, cast(l_orderkey as int)) from lineitem") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + + test("try_divide") { + runQueryAndCompare( + "select try_divide(cast(l_orderkey as int), 0) from lineitem", + noFallBack = false) { + _ => // Spark would always cast inputs to double for this function. + } + } + + testWithSpecifiedSparkVersion("try_multiply", Some("3.3")) { + runQueryAndCompare( + "select try_multiply(2147483647, cast(l_orderkey as int)), " + + "try_multiply(-2147483648, cast(l_orderkey as int)) from lineitem") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("test array forall") { withTempPath { path => diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index b88d781b69b2..b827690d1cdf 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -21,6 +21,7 @@ #include "operators/functions/RowFunctionWithNull.h" #include "velox/expression/SpecialFormRegistry.h" #include "velox/expression/VectorFunction.h" +#include "velox/functions/lib/CheckedArithmetic.h" #include "velox/functions/lib/RegistrationHelpers.h" #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" #include "velox/functions/prestosql/registration/RegistrationFunctions.h" @@ -67,6 +68,10 @@ void registerFunctionOverwrite() { kRowConstructorWithAllNull, std::make_unique(kRowConstructorWithAllNull)); velox::functions::sparksql::registerBitwiseFunctions("spark_"); + velox::functions::registerBinaryIntegral({"check_add"}); + velox::functions::registerBinaryIntegral({"check_subtract"}); + velox::functions::registerBinaryIntegral({"check_multiply"}); + velox::functions::registerBinaryIntegral({"check_divide"}); } } // namespace diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index f417618d8117..0880f3e3d915 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -400,7 +400,6 @@ std::unordered_map SubstraitParser::substraitVeloxFunc {"modulus", "remainder"}, {"date_format", "format_datetime"}, {"collect_set", "set_agg"}, - {"try_add", "plus"}, {"forall", "all_match"}, {"exists", "any_match"}, {"negative", "unaryminus"}, diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index 5d083c4e59ba..f39bd7016707 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -100,7 +100,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | & | bitwise_and | bitwise_and | S | | | | | | | | | | | | | | | | | | | | | * | multiply | multiply | S | ANSI OFF | | S | S | S | S | S | | | | | | | | | | | | | | + | plus | add | S | ANSI OFF | | S | S | S | S | S | | | | | | | | | | | | | -| - | minus | substract | S | ANSI OFF | | S | S | S | S | S | | | | | | | | | | | | | +| - | minus | subtract | S | ANSI OFF | | S | S | S | S | S | | | | | | | | | | | | | | / | divide | divide | S | ANSI OFF | | S | S | S | S | S | | | | | | | | | | | | | | < | lt | lessthan | S | | S | S | S | S | S | S | S | | | S | | | | | | | | | | <= | lte | lessthanorequa | S | | S | S | S | S | S | S | S | | | S | | | | | | | | | diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 78cf02f0ac24..8a086f896ba4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -206,12 +206,13 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(), original) } - def genTryAddTransformer( + def genTryArithmeticTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, - original: TryEval): ExpressionTransformer = { - throw new GlutenNotSupportException("try_add is not supported") + original: TryEval, + checkArithmeticExprName: String): ExpressionTransformer = { + throw new GlutenNotSupportException(s"$checkArithmeticExprName is not supported") } def genTryEvalTransformer( @@ -221,11 +222,12 @@ trait SparkPlanExecApi { throw new GlutenNotSupportException("try_eval is not supported") } - def genAddTransformer( + def genArithmeticTransformer( substraitExprName: String, left: ExpressionTransformer, right: ExpressionTransformer, - original: Add): ExpressionTransformer = { + original: Expression, + checkArithmeticExprName: String): ExpressionTransformer = { GenericExpressionTransformer(substraitExprName, Seq(left, right), original) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index b66ec89eaf2b..9ebe44f6ca54 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -563,18 +563,68 @@ object ExpressionConverter extends SQLConfHelper with Logging { arrayTransform ) case tryEval @ TryEval(a: Add) => - BackendsApiManager.getSparkPlanExecApiInstance.genTryAddTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( substraitExprName, replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), - tryEval + tryEval, + ExpressionNames.CHECK_ADD + ) + case tryEval @ TryEval(a: Subtract) => + BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), + tryEval, + ExpressionNames.CHECK_SUBTRACT + ) + case tryEval @ TryEval(a: Divide) => + BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), + tryEval, + ExpressionNames.CHECK_DIVIDE + ) + case tryEval @ TryEval(a: Multiply) => + BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), + tryEval, + ExpressionNames.CHECK_MULTIPLY ) case a: Add => - BackendsApiManager.getSparkPlanExecApiInstance.genAddTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( substraitExprName, replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), - a + a, + ExpressionNames.CHECK_ADD + ) + case a: Subtract => + BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), + a, + ExpressionNames.CHECK_SUBTRACT + ) + case a: Multiply => + BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), + a, + ExpressionNames.CHECK_MULTIPLY + ) + case a: Divide => + BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), + replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), + a, + ExpressionNames.CHECK_DIVIDE ) case tryEval: TryEval => // This is a placeholder to handle try_eval(other expressions). diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index d8e3a5ecc051..bd437bbe8efb 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -20,7 +20,7 @@ import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} import org.apache.spark.GlutenSortShuffleSuite import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} +import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite, GlutenTryEvalSuite} import org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite, GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite, GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter, GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite, GlutenDeleteFromTableSuite, GlutenDeltaBasedDeleteFromTableSuite, GlutenFileDataSourceV2FallBackSuite, GlutenGroupBasedDeleteFromTableSuite, GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, GlutenMetadataColumnSuite, GlutenSupportsCatalogOptionsSuite, GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite} import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} import org.apache.spark.sql.execution.{FallbackStrategiesSuite, GlutenBroadcastExchangeSuite, GlutenCoalesceShufflePartitionsSuite, GlutenExchangeSuite, GlutenLocalBroadcastExchangeSuite, GlutenReplaceHashWithSortAggSuite, GlutenReuseExchangeAndSubquerySuite, GlutenSameResultSuite, GlutenSortSuite, GlutenSQLAggregateFunctionSuite, GlutenSQLWindowFunctionSuite, GlutenTakeOrderedAndProjectSuite} @@ -141,6 +141,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSortShuffleSuite] enableSuite[GlutenSortOrderExpressionsSuite] enableSuite[GlutenStringExpressionsSuite] + enableSuite[GlutenTryEvalSuite] enableSuite[VeloxAdaptiveQueryExecSuite] .includeAllGlutenTests() .includeByPrefix( diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala new file mode 100644 index 000000000000..6af97677e5d8 --- /dev/null +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.spark.sql.GlutenTestsTrait + +class GlutenTryEvalSuite extends TryEvalSuite with GlutenTestsTrait {} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 10f7be4feaeb..af8d0deadfc8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -20,7 +20,7 @@ import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings} import org.apache.spark.GlutenSortShuffleSuite import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite} +import org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite, GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite, GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite, GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite, GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite, GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite, GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, GlutenNondeterministicSuite, GlutenNullExpressionsSuite, GlutenPredicateSuite, GlutenRandomSuite, GlutenRegexpExpressionsSuite, GlutenSortOrderExpressionsSuite, GlutenStringExpressionsSuite, GlutenTryEvalSuite} import org.apache.spark.sql.connector._ import org.apache.spark.sql.errors.{GlutenQueryCompilationErrorsDSv2Suite, GlutenQueryCompilationErrorsSuite, GlutenQueryExecutionErrorsSuite, GlutenQueryParsingErrorsSuite} import org.apache.spark.sql.execution._ @@ -144,6 +144,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSortShuffleSuite] enableSuite[GlutenSortOrderExpressionsSuite] enableSuite[GlutenStringExpressionsSuite] + enableSuite[GlutenTryEvalSuite] enableSuite[VeloxAdaptiveQueryExecSuite] .includeAllGlutenTests() .includeByPrefix( diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala new file mode 100644 index 000000000000..6af97677e5d8 --- /dev/null +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryEvalSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.spark.sql.GlutenTestsTrait + +class GlutenTryEvalSuite extends TryEvalSuite with GlutenTestsTrait {} diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index be7e32fc97d6..dc98f31a395c 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -83,7 +83,10 @@ object ExpressionNames { final val IS_NAN = "isnan" final val NANVL = "nanvl" final val TRY_EVAL = "try" - final val TRY_ADD = "try_add" + final val CHECK_ADD = "check_add" + final val CHECK_SUBTRACT = "check_subtract" + final val CHECK_DIVIDE = "check_divide" + final val CHECK_MULTIPLY = "check_multiply" // SparkSQL String functions final val ASCII = "ascii" diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 4ab307e8568f..f2c2482949b7 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -407,6 +407,9 @@ class Spark34Shims extends SparkShims { override def withTryEvalMode(expr: Expression): Boolean = { expr match { case a: Add => a.evalMode == EvalMode.TRY + case s: Subtract => s.evalMode == EvalMode.TRY + case d: Divide => d.evalMode == EvalMode.TRY + case m: Multiply => m.evalMode == EvalMode.TRY case _ => false } } @@ -414,6 +417,9 @@ class Spark34Shims extends SparkShims { override def withAnsiEvalMode(expr: Expression): Boolean = { expr match { case a: Add => a.evalMode == EvalMode.ANSI + case s: Subtract => s.evalMode == EvalMode.ANSI + case d: Divide => d.evalMode == EvalMode.ANSI + case m: Multiply => m.evalMode == EvalMode.ANSI case _ => false } } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index ef1cea865d49..e0835c3069d2 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -436,6 +436,9 @@ class Spark35Shims extends SparkShims { override def withTryEvalMode(expr: Expression): Boolean = { expr match { case a: Add => a.evalMode == EvalMode.TRY + case s: Subtract => s.evalMode == EvalMode.TRY + case d: Divide => d.evalMode == EvalMode.TRY + case m: Multiply => m.evalMode == EvalMode.TRY case _ => false } } @@ -443,6 +446,9 @@ class Spark35Shims extends SparkShims { override def withAnsiEvalMode(expr: Expression): Boolean = { expr match { case a: Add => a.evalMode == EvalMode.ANSI + case s: Subtract => s.evalMode == EvalMode.ANSI + case d: Divide => d.evalMode == EvalMode.ANSI + case m: Multiply => m.evalMode == EvalMode.ANSI case _ => false } } From d1b3e9918fcd087f348d0faf787e42246650b502 Mon Sep 17 00:00:00 2001 From: lgbo Date: Thu, 6 Jun 2024 13:43:20 +0800 Subject: [PATCH 221/402] fixed missing columns when there is mixed join conditions (#5997) --- .../gluten/vectorized/StorageJoinBuilder.java | 2 + .../execution/CHHashJoinExecTransformer.scala | 20 ++++++- ...enClickHouseTPCHSaltNullParquetSuite.scala | 12 ++++- .../benchmarks/CHHashBuildBenchmark.scala | 2 +- .../Join/BroadCastJoinBuilder.cpp | 2 + .../local-engine/Join/BroadCastJoinBuilder.h | 1 + .../Join/StorageJoinFromReadBuffer.cpp | 54 ++++++++++++++++++- .../Join/StorageJoinFromReadBuffer.h | 6 +++ cpp-ch/local-engine/local_engine_jni.cpp | 14 +++-- 9 files changed, 104 insertions(+), 9 deletions(-) diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java index 065be9de2557..9cb49b6a2d30 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/StorageJoinBuilder.java @@ -44,6 +44,7 @@ private static native long nativeBuild( long rowCount, String joinKeys, int joinType, + boolean hasMixedFiltCondition, byte[] namedStruct); private StorageJoinBuilder() {} @@ -79,6 +80,7 @@ public static long build( rowCount, joinKey, SubstraitUtil.toSubstrait(broadCastContext.joinType()).ordinal(), + broadCastContext.hasMixedFiltCondition(), toNameStruct(output).toByteArray()); } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala index 6004f7f861bf..a7e7769e7736 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala @@ -82,6 +82,7 @@ case class CHBroadcastBuildSideRDD( case class BroadCastHashJoinContext( buildSideJoinKeys: Seq[Expression], joinType: JoinType, + hasMixedFiltCondition: Boolean, buildSideStructure: Seq[Attribute], buildHashTableId: String) @@ -139,9 +140,26 @@ case class CHBroadcastHashJoinExecTransformer( } val broadcast = buildPlan.executeBroadcast[BuildSideRelation]() val context = - BroadCastHashJoinContext(buildKeyExprs, joinType, buildPlan.output, buildHashTableId) + BroadCastHashJoinContext( + buildKeyExprs, + joinType, + isMixedCondition(condition), + buildPlan.output, + buildHashTableId) val broadcastRDD = CHBroadcastBuildSideRDD(sparkContext, broadcast, context) // FIXME: Do we have to make build side a RDD? streamedRDD :+ broadcastRDD } + + def isMixedCondition(cond: Option[Expression]): Boolean = { + val res = if (cond.isDefined) { + val leftOutputSet = left.outputSet + val rightOutputSet = right.outputSet + val allReferences = cond.get.references + !(allReferences.subsetOf(leftOutputSet) || allReferences.subsetOf(rightOutputSet)) + } else { + false + } + res + } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index ada980a20bc2..ee495457edee 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2593,13 +2593,21 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("create table ineq_join_t2 (key bigint, value bigint) using parquet"); spark.sql("insert into ineq_join_t1 values(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)"); spark.sql("insert into ineq_join_t2 values(2, 2), (2, 1), (3, 3), (4, 6), (5, 3)"); - val sql = + val sql1 = """ | select t1.key, t1.value, t2.key, t2.value from ineq_join_t1 as t1 | left join ineq_join_t2 as t2 | on t1.key = t2.key and t1.value > t2.value |""".stripMargin - compareResultsAgainstVanillaSpark(sql, true, { _ => }) + compareResultsAgainstVanillaSpark(sql1, true, { _ => }) + + val sql2 = + """ + | select t1.key, t1.value from ineq_join_t1 as t1 + | left join ineq_join_t2 as t2 + | on t1.key = t2.key and t1.value > t2.value and t1.value > t2.key + |""".stripMargin + compareResultsAgainstVanillaSpark(sql2, true, { _ => }) spark.sql("drop table ineq_join_t1") spark.sql("drop table ineq_join_t2") } diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHHashBuildBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHHashBuildBenchmark.scala index 487433c469c1..8d4bee554625 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHHashBuildBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHHashBuildBenchmark.scala @@ -104,7 +104,7 @@ object CHHashBuildBenchmark extends SqlBasedBenchmark with CHSqlBasedBenchmark w ( countsAndBytes.flatMap(_._2), countsAndBytes.map(_._1).sum, - BroadCastHashJoinContext(Seq(child.output.head), Inner, child.output, "") + BroadCastHashJoinContext(Seq(child.output.head), Inner, false, child.output, "") ) } } diff --git a/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp b/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp index f1b3ac2fbd9c..1c79a00a7c4c 100644 --- a/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp +++ b/cpp-ch/local-engine/Join/BroadCastJoinBuilder.cpp @@ -82,6 +82,7 @@ std::shared_ptr buildJoin( jlong row_count, const std::string & join_keys, substrait::JoinRel_JoinType join_type, + bool has_mixed_join_condition, const std::string & named_struct) { auto join_key_list = Poco::StringTokenizer(join_keys, ","); @@ -105,6 +106,7 @@ std::shared_ptr buildJoin( true, kind, strictness, + has_mixed_join_condition, columns_description, ConstraintsDescription(), key, diff --git a/cpp-ch/local-engine/Join/BroadCastJoinBuilder.h b/cpp-ch/local-engine/Join/BroadCastJoinBuilder.h index 5aa1e0876ed0..9a6837e35a0a 100644 --- a/cpp-ch/local-engine/Join/BroadCastJoinBuilder.h +++ b/cpp-ch/local-engine/Join/BroadCastJoinBuilder.h @@ -36,6 +36,7 @@ std::shared_ptr buildJoin( jlong row_count, const std::string & join_keys, substrait::JoinRel_JoinType join_type, + bool has_mixed_join_condition, const std::string & named_struct); void cleanBuildHashTable(const std::string & hash_table_id, jlong instance); std::shared_ptr getJoin(const std::string & hash_table_id); diff --git a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp index f0aec6af686d..af306564a4c5 100644 --- a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp +++ b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp @@ -74,6 +74,7 @@ StorageJoinFromReadBuffer::StorageJoinFromReadBuffer( bool use_nulls_, DB::JoinKind kind, DB::JoinStrictness strictness, + bool has_mixed_join_condition, const ColumnsDescription & columns, const ConstraintsDescription & constraints, const String & comment, @@ -91,7 +92,11 @@ StorageJoinFromReadBuffer::StorageJoinFromReadBuffer( key_names.push_back(RIHGT_COLUMN_PREFIX + name); auto table_join = std::make_shared(SizeLimits(), true, kind, strictness, key_names); right_sample_block = rightSampleBlock(use_nulls, storage_metadata, table_join->kind()); - buildJoin(in, right_sample_block, table_join); + /// If there is mixed join conditions, need to build the hash join lazily, which rely on the real table join. + if (!has_mixed_join_condition) + buildJoin(in, right_sample_block, table_join); + else + collectAllInputs(in, right_sample_block); } /// The column names may be different in two blocks. @@ -135,6 +140,51 @@ void StorageJoinFromReadBuffer::buildJoin(DB::ReadBuffer & in, const Block heade } } +void StorageJoinFromReadBuffer::collectAllInputs(DB::ReadBuffer & in, const DB::Block header) +{ + local_engine::NativeReader block_stream(in); + ProfileInfo info; + while (Block block = block_stream.read()) + { + DB::ColumnsWithTypeAndName columns; + for (size_t i = 0; i < block.columns(); ++i) + { + const auto & column = block.getByPosition(i); + columns.emplace_back(convertColumnAsNecessary(column, header.getByPosition(i))); + } + DB::Block final_block(columns); + info.update(final_block); + input_blocks.emplace_back(std::move(final_block)); + } +} + +void StorageJoinFromReadBuffer::buildJoinLazily(DB::Block header, std::shared_ptr analyzed_join) +{ + { + std::shared_lock lock(join_mutex); + if (join) + return; + } + std::unique_lock lock(join_mutex); + if (join) + return; + join = std::make_shared(analyzed_join, header, overwrite, row_count); + while(!input_blocks.empty()) + { + auto & block = *input_blocks.begin(); + DB::ColumnsWithTypeAndName columns; + for (size_t i = 0; i < block.columns(); ++i) + { + const auto & column = block.getByPosition(i); + columns.emplace_back(convertColumnAsNecessary(column, header.getByPosition(i))); + } + DB::Block final_block(columns); + join->addBlockToJoin(final_block, true); + input_blocks.pop_front(); + } +} + + /// The column names of 'rgiht_header' could be different from the ones in `input_blocks`, and we must /// use 'right_header' to build the HashJoin. Otherwise, it will cause exceptions with name mismatches. /// @@ -148,7 +198,7 @@ DB::JoinPtr StorageJoinFromReadBuffer::getJoinLocked(std::shared_ptr(analyzed_join, right_sample_block); /// reuseJoinedData will set the flag `HashJoin::from_storage_join` which is required by `FilledStep` join_clone->reuseJoinedData(static_cast(*join)); diff --git a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h index af623c0cd717..ddefda69c30f 100644 --- a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h +++ b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.h @@ -15,6 +15,7 @@ * limitations under the License. */ #pragma once +#include #include #include @@ -40,6 +41,7 @@ class StorageJoinFromReadBuffer bool use_nulls_, DB::JoinKind kind, DB::JoinStrictness strictness, + bool has_mixed_join_condition, const DB::ColumnsDescription & columns_, const DB::ConstraintsDescription & constraints_, const String & comment, @@ -58,9 +60,13 @@ class StorageJoinFromReadBuffer size_t row_count; bool overwrite; DB::Block right_sample_block; + std::shared_mutex join_mutex; + std::list input_blocks; std::shared_ptr join = nullptr; void readAllBlocksFromInput(DB::ReadBuffer & in); void buildJoin(DB::ReadBuffer & in, const DB::Block header, std::shared_ptr analyzed_join); + void collectAllInputs(DB::ReadBuffer & in, const DB::Block header); + void buildJoinLazily(DB::Block header, std::shared_ptr analyzed_join); }; } diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index be28b9fabeff..38f188293726 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -1172,7 +1172,15 @@ JNIEXPORT jobject Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn } JNIEXPORT jlong Java_org_apache_gluten_vectorized_StorageJoinBuilder_nativeBuild( - JNIEnv * env, jclass, jstring key, jbyteArray in, jlong row_count_, jstring join_key_, jint join_type_, jbyteArray named_struct) + JNIEnv * env, + jclass, + jstring key, + jbyteArray in, + jlong row_count_, + jstring join_key_, + jint join_type_, + jboolean has_mixed_join_condition, + jbyteArray named_struct) { LOCAL_ENGINE_JNI_METHOD_START const auto hash_table_id = jstring2string(env, key); @@ -1186,8 +1194,8 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_StorageJoinBuilder_nativeBuild local_engine::ReadBufferFromByteArray read_buffer_from_java_array(in, length); DB::CompressedReadBuffer input(read_buffer_from_java_array); local_engine::configureCompressedReadBuffer(input); - const auto * obj - = make_wrapper(local_engine::BroadCastJoinBuilder::buildJoin(hash_table_id, input, row_count_, join_key, join_type, struct_string)); + const auto * obj = make_wrapper(local_engine::BroadCastJoinBuilder::buildJoin( + hash_table_id, input, row_count_, join_key, join_type, has_mixed_join_condition, struct_string)); env->ReleaseByteArrayElements(named_struct, struct_address, JNI_ABORT); return obj->instance(); LOCAL_ENGINE_JNI_METHOD_END(env, 0) From f72349ed8b18b40b45428a2c11bb658988c8e97c Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 6 Jun 2024 15:32:39 +0800 Subject: [PATCH 222/402] [VL] Make ColumnarBatch::getRowBytes leak-safe (#6002) --- cpp/core/jni/JniWrapper.cc | 29 ++++++++++---------------- cpp/core/memory/ColumnarBatch.cc | 16 +++++++------- cpp/core/memory/ColumnarBatch.h | 9 ++++---- cpp/velox/memory/VeloxColumnarBatch.cc | 10 ++++----- cpp/velox/memory/VeloxColumnarBatch.h | 2 +- 5 files changed, 30 insertions(+), 36 deletions(-) diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index f5a6c4bd70d0..db498f43adbf 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -72,8 +72,8 @@ static jclass shuffleReaderMetricsClass; static jmethodID shuffleReaderMetricsSetDecompressTime; static jmethodID shuffleReaderMetricsSetDeserializeTime; -static jclass block_stripes_class; -static jmethodID block_stripes_constructor; +static jclass blockStripesClass; +static jmethodID blockStripesConstructor; class JavaInputStreamAdaptor final : public arrow::io::InputStream { public: @@ -280,9 +280,9 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { shuffleReaderMetricsSetDeserializeTime = getMethodIdOrError(env, shuffleReaderMetricsClass, "setDeserializeTime", "(J)V"); - block_stripes_class = + blockStripesClass = createGlobalClassReferenceOrError(env, "Lorg/apache/spark/sql/execution/datasources/BlockStripes;"); - block_stripes_constructor = env->GetMethodID(block_stripes_class, "", "(J[J[II[B)V"); + blockStripesConstructor = env->GetMethodID(blockStripesClass, "", "(J[J[II[B)V"); return jniVersion; } @@ -297,7 +297,7 @@ void JNI_OnUnload(JavaVM* vm, void* reserved) { env->DeleteGlobalRef(nativeColumnarToRowInfoClass); env->DeleteGlobalRef(byteArrayClass); env->DeleteGlobalRef(shuffleReaderMetricsClass); - env->DeleteGlobalRef(block_stripes_class); + env->DeleteGlobalRef(blockStripesClass); gluten::getJniErrorState()->close(); gluten::getJniCommonState()->close(); @@ -1224,14 +1224,13 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB } MemoryManager* memoryManager = reinterpret_cast(memoryManagerId); - auto result = batch->getRowBytes(0); - auto rowBytes = result.first; + auto result = batch->toUnsafeRow(0); + auto rowBytes = result.data(); auto newBatchHandle = ctx->objectStore()->save(ctx->select(memoryManager, batch, partitionColIndiceVec)); - auto bytesSize = result.second; + auto bytesSize = result.size(); jbyteArray bytesArray = env->NewByteArray(bytesSize); env->SetByteArrayRegion(bytesArray, 0, bytesSize, reinterpret_cast(rowBytes)); - delete[] rowBytes; jlongArray batchArray = env->NewLongArray(1); long* cBatchArray = new long[1]; @@ -1239,15 +1238,9 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB env->SetLongArrayRegion(batchArray, 0, 1, cBatchArray); delete[] cBatchArray; - jobject block_stripes = env->NewObject( - block_stripes_class, - block_stripes_constructor, - batchHandle, - batchArray, - nullptr, - batch->numColumns(), - bytesArray); - return block_stripes; + jobject blockStripes = env->NewObject( + blockStripesClass, blockStripesConstructor, batchHandle, batchArray, nullptr, batch->numColumns(), bytesArray); + return blockStripes; JNI_METHOD_END(nullptr) } diff --git a/cpp/core/memory/ColumnarBatch.cc b/cpp/core/memory/ColumnarBatch.cc index bb80510ee351..23567535d50a 100644 --- a/cpp/core/memory/ColumnarBatch.cc +++ b/cpp/core/memory/ColumnarBatch.cc @@ -43,8 +43,8 @@ int64_t ColumnarBatch::getExportNanos() const { return exportNanos_; } -std::pair ColumnarBatch::getRowBytes(int32_t rowId) const { - throw gluten::GlutenException("Not implemented getRowBytes for ColumnarBatch"); +std::vector ColumnarBatch::toUnsafeRow(int32_t rowId) const { + throw gluten::GlutenException("Not implemented toUnsafeRow for ColumnarBatch"); } std::ostream& operator<<(std::ostream& os, const ColumnarBatch& columnarBatch) { @@ -86,8 +86,8 @@ std::shared_ptr ArrowColumnarBatch::exportArrowArray() { return cArray; } -std::pair ArrowColumnarBatch::getRowBytes(int32_t rowId) const { - throw gluten::GlutenException("Not implemented getRowBytes for ArrowColumnarBatch"); +std::vector ArrowColumnarBatch::toUnsafeRow(int32_t rowId) const { + throw gluten::GlutenException("#toUnsafeRow of ArrowColumnarBatch is not implemented"); } ArrowCStructColumnarBatch::ArrowCStructColumnarBatch( @@ -123,8 +123,8 @@ std::shared_ptr ArrowCStructColumnarBatch::exportArrowArray() { return cArray_; } -std::pair ArrowCStructColumnarBatch::getRowBytes(int32_t rowId) const { - throw gluten::GlutenException("Not implemented getRowBytes for ArrowCStructColumnarBatch"); +std::vector ArrowCStructColumnarBatch::toUnsafeRow(int32_t rowId) const { + throw gluten::GlutenException("#toUnsafeRow of ArrowCStructColumnarBatch is not implemented"); } std::shared_ptr CompositeColumnarBatch::create(std::vector> batches) { @@ -171,8 +171,8 @@ const std::vector>& CompositeColumnarBatch::getBa return batches_; } -std::pair CompositeColumnarBatch::getRowBytes(int32_t rowId) const { - throw gluten::GlutenException("Not implemented getRowBytes for CompositeColumnarBatch"); +std::vector CompositeColumnarBatch::toUnsafeRow(int32_t rowId) const { + throw gluten::GlutenException("#toUnsafeRow of CompositeColumnarBatch is not implemented"); } CompositeColumnarBatch::CompositeColumnarBatch( diff --git a/cpp/core/memory/ColumnarBatch.h b/cpp/core/memory/ColumnarBatch.h index 4a7b34889f60..fd8189aa6a20 100644 --- a/cpp/core/memory/ColumnarBatch.h +++ b/cpp/core/memory/ColumnarBatch.h @@ -49,7 +49,8 @@ class ColumnarBatch { virtual int64_t getExportNanos() const; - virtual std::pair getRowBytes(int32_t rowId) const; + // Serializes one single row to byte array that can be accessed as Spark-compatible unsafe row. + virtual std::vector toUnsafeRow(int32_t rowId) const; friend std::ostream& operator<<(std::ostream& os, const ColumnarBatch& columnarBatch); @@ -75,7 +76,7 @@ class ArrowColumnarBatch final : public ColumnarBatch { std::shared_ptr exportArrowArray() override; - std::pair getRowBytes(int32_t rowId) const override; + std::vector toUnsafeRow(int32_t rowId) const override; private: std::shared_ptr batch_; @@ -95,7 +96,7 @@ class ArrowCStructColumnarBatch final : public ColumnarBatch { std::shared_ptr exportArrowArray() override; - std::pair getRowBytes(int32_t rowId) const override; + std::vector toUnsafeRow(int32_t rowId) const override; private: std::shared_ptr cSchema_ = std::make_shared(); @@ -120,7 +121,7 @@ class CompositeColumnarBatch final : public ColumnarBatch { const std::vector>& getBatches() const; - std::pair getRowBytes(int32_t rowId) const override; + std::vector toUnsafeRow(int32_t rowId) const override; private: explicit CompositeColumnarBatch( diff --git a/cpp/velox/memory/VeloxColumnarBatch.cc b/cpp/velox/memory/VeloxColumnarBatch.cc index 83428707b320..0d8db312721a 100644 --- a/cpp/velox/memory/VeloxColumnarBatch.cc +++ b/cpp/velox/memory/VeloxColumnarBatch.cc @@ -143,13 +143,13 @@ std::shared_ptr VeloxColumnarBatch::select( return std::make_shared(rowVector); } -std::pair VeloxColumnarBatch::getRowBytes(int32_t rowId) const { +std::vector VeloxColumnarBatch::toUnsafeRow(int32_t rowId) const { auto fast = std::make_unique(rowVector_); auto size = fast->rowSize(rowId); - char* rowBytes = new char[size]; - std::memset(rowBytes, 0, size); - fast->serialize(0, rowBytes); - return std::make_pair(rowBytes, size); + std::vector bytes(size); + std::memset(bytes.data(), 0, bytes.size()); + fast->serialize(0, bytes.data()); + return bytes; } } // namespace gluten diff --git a/cpp/velox/memory/VeloxColumnarBatch.h b/cpp/velox/memory/VeloxColumnarBatch.h index c319b7977c33..6c79f2772d2d 100644 --- a/cpp/velox/memory/VeloxColumnarBatch.h +++ b/cpp/velox/memory/VeloxColumnarBatch.h @@ -41,7 +41,7 @@ class VeloxColumnarBatch final : public ColumnarBatch { std::shared_ptr exportArrowSchema() override; std::shared_ptr exportArrowArray() override; - std::pair getRowBytes(int32_t rowId) const override; + std::vector toUnsafeRow(int32_t rowId) const override; std::shared_ptr select(facebook::velox::memory::MemoryPool* pool, std::vector columnIndices); facebook::velox::RowVectorPtr getRowVector() const; facebook::velox::RowVectorPtr getFlattenedRowVector(); From c23d28db779622fb2186002cc89e06fa5e24e5f4 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Thu, 6 Jun 2024 02:41:03 -0500 Subject: [PATCH 223/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240606) (#5999) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240606) 1. Fix UT due to https://github.com/ClickHouse/ClickHouse/pull/63723 2. Fix UT in Debug mode 3. Reopen UT --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- .../gluten/compatibility/GlutenFunctionSuite.scala | 12 ++++-------- cpp-ch/clickhouse.version | 4 ++-- cpp-ch/local-engine/tests/gtest_ch_functions.cpp | 14 ++++++++++---- .../tests/gtest_parquet_columnindex.cpp | 4 +++- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala index d0e13b49609a..aaee8241206e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala @@ -46,8 +46,7 @@ class GlutenFunctionSuite | , left(`101`, 0) | , left(`101`, -1) -- error | from parquet.`$testPath/left` - |""".stripMargin, - ignore = true + |""".stripMargin ), TestCase( "trim", @@ -56,8 +55,7 @@ class GlutenFunctionSuite | , trim(LEADING `100` from `99`) -- error | , trim(TRAILING `100` from `99`) -- error | from parquet.`$testPath/left` - |""".stripMargin, - ignore = true + |""".stripMargin ), TestCase( "date_format 1", @@ -72,8 +70,7 @@ class GlutenFunctionSuite | , date_format(`0`, 'm') | , date_format(`0`, 's') | from parquet.`$testPath/date_format/date` - |""".stripMargin, - ignore = true + |""".stripMargin ), TestCase( "date_format 2", @@ -88,8 +85,7 @@ class GlutenFunctionSuite | , date_format(`4`, 'm') | , date_format(`4`, 's') | from parquet.`$testPath/date_format/timestamp` - |""".stripMargin, - ignore = true + |""".stripMargin ) ) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index c1baa6037996..d98ce9f8852a 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240527 -CH_COMMIT=1388dcb5b0bbb630af259280f4287e3342ca6237 +CH_BRANCH=rebase_ch/20240606 +CH_COMMIT=fed1c01e169 diff --git a/cpp-ch/local-engine/tests/gtest_ch_functions.cpp b/cpp-ch/local-engine/tests/gtest_ch_functions.cpp index 943826287d54..613beb9b8051 100644 --- a/cpp-ch/local-engine/tests/gtest_ch_functions.cpp +++ b/cpp-ch/local-engine/tests/gtest_ch_functions.cpp @@ -79,10 +79,12 @@ TEST(TestFunction, In) set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName()); set->finishInsert(); auto future_set = std::make_shared(std::move(set)); - auto arg = ColumnSet::create(1, future_set); + //TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we need pass 4 instead of 1 + auto arg = ColumnSet::create(4, future_set); ColumnsWithTypeAndName columns - = {ColumnWithTypeAndName(std::move(column1), type0, "string0"), ColumnWithTypeAndName(std::move(arg), type_set, "__set")}; + = {ColumnWithTypeAndName(std::move(column1), type0, "string0"), + ColumnWithTypeAndName(std::move(arg), type_set, "__set")}; Block block(columns); std::cerr << "input:\n"; debug::headBlock(block); @@ -121,7 +123,9 @@ TEST(TestFunction, NotIn1) set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName()); set->finishInsert(); auto future_set = std::make_shared(std::move(set)); - auto arg = ColumnSet::create(1,future_set); + + //TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we need pass 4 instead of 1 + auto arg = ColumnSet::create(4,future_set); ColumnsWithTypeAndName columns = {ColumnWithTypeAndName(std::move(column1), type0, "string0"), ColumnWithTypeAndName(std::move(arg), type_set, "__set")}; @@ -162,7 +166,9 @@ TEST(TestFunction, NotIn2) set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName()); set->finishInsert(); auto future_set = std::make_shared(std::move(set)); - auto arg = ColumnSet::create(1,future_set); + + //TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we need pass 4 instead of 1 + auto arg = ColumnSet::create(4,future_set); ColumnsWithTypeAndName columns = {ColumnWithTypeAndName(std::move(column1), type0, "string0"), ColumnWithTypeAndName(std::move(arg), type_set, "__set")}; diff --git a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp index ea3bd41e4384..532244029b78 100644 --- a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp +++ b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp @@ -604,13 +604,15 @@ TEST(ColumnIndex, DecimalField) ASSERT_EQ(actual, expected); - /// Eexception test + /// Eexception test, only in relase release node +#ifdef NDEBUG Field unsupport = DecimalField(Int256(300000000), 4); EXPECT_THROW(to_parquet.as(unsupport, desc), DB::Exception); const parquet::ColumnDescriptor error = PNB::optional(parquet::Type::FIXED_LEN_BYTE_ARRAY).asDecimal(38, 4).with_length(18).descriptor("column1"); EXPECT_THROW(to_parquet.as(value, error), DB::Exception); +#endif } From 60b8aadf41b668847e750c23c93ab263a4530503 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 6 Jun 2024 19:51:16 +0800 Subject: [PATCH 224/402] [VL] Daily Update Velox Version (2024_06_05) (#5998) Co-authored-by: PHILO-HE --- cpp/velox/CMakeLists.txt | 12 +++++++++++- cpp/velox/memory/VeloxMemoryManager.cc | 10 +++++----- cpp/velox/operators/writer/VeloxParquetDatasource.cc | 2 +- cpp/velox/tests/BufferOutputStreamTest.cc | 6 +++--- ep/build-velox/src/get_velox.sh | 2 +- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 9bedfe45ba0e..05ecf9635eb0 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -128,6 +128,7 @@ macro(ADD_VELOX_DEPENDENCIES) add_velox_dependency(functions::json "${VELOX_COMPONENTS_PATH}/functions/prestosql/json/libvelox_functions_json.a") add_velox_dependency(functions::hyperloglog "${VELOX_COMPONENTS_PATH}/common/hyperloglog/libvelox_common_hyperloglog.a") add_velox_dependency(functions::lib "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib.a") + add_velox_dependency(functions::lib::date_time_formatter "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib_date_time_formatter.a") if(BUILD_TESTS) add_velox_dependency(exec::test "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_exec_test_lib.a") add_velox_dependency(temp::path "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_temp_path.a") @@ -230,7 +231,6 @@ macro(ADD_VELOX_DEPENDENCIES) add_velox_dependency(common::compression "${VELOX_COMPONENTS_PATH}/common/compression/libvelox_common_compression.a") add_velox_dependency(common::io "${VELOX_COMPONENTS_PATH}/common/io/libvelox_common_io.a") add_velox_dependency(velox::status "${VELOX_COMPONENTS_PATH}/common/base/libvelox_status.a") - add_velox_dependency(external::simdjson "${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a") endmacro() macro(find_libhdfs3) @@ -389,6 +389,16 @@ else() add_velox_dependency(velox "${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a") endif() +set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) +set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") +find_package(simdjson CONFIG) +if(simdjson_FOUND AND TARGET simdjson::simdjson) + target_link_libraries(velox PUBLIC simdjson::simdjson) +else() + add_velox_dependency(external::simdjson "${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a") +endif() +set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) + if(ENABLE_GLUTEN_VCPKG) find_package(Thrift CONFIG) else() diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 496ebf4522e1..60c79ffe8725 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -195,7 +195,7 @@ VeloxMemoryManager::VeloxMemoryManager( namespace { MemoryUsageStats collectVeloxMemoryUsageStats(const velox::memory::MemoryPool* pool) { MemoryUsageStats stats; - stats.set_current(pool->currentBytes()); + stats.set_current(pool->usedBytes()); stats.set_peak(pool->peakBytes()); // walk down root and all children pool->visitChildren([&](velox::memory::MemoryPool* pool) -> bool { @@ -216,7 +216,7 @@ int64_t shrinkVeloxMemoryPool(velox::memory::MemoryManager* mm, velox::memory::M std::string poolName{pool->root()->name() + "/" + pool->name()}; std::string logPrefix{"Shrink[" + poolName + "]: "}; VLOG(2) << logPrefix << "Trying to shrink " << size << " bytes of data..."; - VLOG(2) << logPrefix << "Pool has reserved " << pool->currentBytes() << "/" << pool->root()->reservedBytes() << "/" + VLOG(2) << logPrefix << "Pool has reserved " << pool->usedBytes() << "/" << pool->root()->reservedBytes() << "/" << pool->root()->capacity() << "/" << pool->root()->maxCapacity() << " bytes."; VLOG(2) << logPrefix << "Shrinking..."; const uint64_t oldCapacity = pool->capacity(); @@ -263,14 +263,14 @@ void VeloxMemoryManager::hold() { bool VeloxMemoryManager::tryDestructSafe() { // Velox memory pools considered safe to destruct when no alive allocations. for (const auto& pool : heldVeloxPools_) { - if (pool && pool->currentBytes() != 0) { + if (pool && pool->usedBytes() != 0) { return false; } } - if (veloxLeafPool_ && veloxLeafPool_->currentBytes() != 0) { + if (veloxLeafPool_ && veloxLeafPool_->usedBytes() != 0) { return false; } - if (veloxAggregatePool_ && veloxAggregatePool_->currentBytes() != 0) { + if (veloxAggregatePool_ && veloxAggregatePool_->usedBytes() != 0) { return false; } heldVeloxPools_.clear(); diff --git a/cpp/velox/operators/writer/VeloxParquetDatasource.cc b/cpp/velox/operators/writer/VeloxParquetDatasource.cc index 16558229e765..58aa9f33af04 100644 --- a/cpp/velox/operators/writer/VeloxParquetDatasource.cc +++ b/cpp/velox/operators/writer/VeloxParquetDatasource.cc @@ -120,7 +120,7 @@ void VeloxParquetDatasource::inspectSchema(struct ArrowSchema* out) { std::shared_ptr readFile{fs->openFileForRead(filePath_)}; std::unique_ptr reader = - velox::dwio::common::getReaderFactory(readerOptions.getFileFormat()) + velox::dwio::common::getReaderFactory(readerOptions.fileFormat()) ->createReader( std::make_unique( std::make_shared(readFile), *pool_.get()), diff --git a/cpp/velox/tests/BufferOutputStreamTest.cc b/cpp/velox/tests/BufferOutputStreamTest.cc index 3b3f78ceaefe..324d8c5e6394 100644 --- a/cpp/velox/tests/BufferOutputStreamTest.cc +++ b/cpp/velox/tests/BufferOutputStreamTest.cc @@ -55,16 +55,16 @@ TEST_F(BufferOutputStreamTest, outputStream) { reference->write(data.data(), data.size()); } auto str = referenceSStream.str(); - auto numBytes = veloxPool_->currentBytes(); + auto numBytes = veloxPool_->usedBytes(); EXPECT_LT(0, numBytes); { auto buffer = out->getBuffer(); - EXPECT_EQ(numBytes, veloxPool_->currentBytes()); + EXPECT_EQ(numBytes, veloxPool_->usedBytes()); EXPECT_EQ(str, std::string(buffer->as(), buffer->size())); } out.reset(); // We expect dropping the stream frees the backing memory. - EXPECT_EQ(0, veloxPool_->currentBytes()); + EXPECT_EQ(0, veloxPool_->usedBytes()); } } // namespace gluten diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 5aa3f2b379c0..584c9d40d5b9 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_04 +VELOX_BRANCH=2024_06_05 VELOX_HOME="" #Set on run gluten on HDFS From 17697fa6e72de53cc9aa7c3c8b99b0f17f679f9e Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Fri, 7 Jun 2024 04:19:33 +0530 Subject: [PATCH 225/402] Update script (#6006) Update formatcppcode.sh to install clang 15 if not present. --- dev/formatcppcode.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dev/formatcppcode.sh b/dev/formatcppcode.sh index d391235dac9d..4ce4155f78b6 100755 --- a/dev/formatcppcode.sh +++ b/dev/formatcppcode.sh @@ -1,3 +1,13 @@ cd `dirname $0` + +# Check if clang-format-15 is installed +if ! command -v clang-format-15 &> /dev/null +then + echo "clang-format-15 could not be found" + echo "Installing clang-format-15..." + sudo apt update + sudo apt install clang-format-15 +fi + find ../cpp/core -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-15 -style=file -i {} \; -find ../cpp/velox -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-15 -style=file -i {} \; +find ../cpp/velox -regex '.*\.\(cc\|hpp\|cu\|c\|h\)' -exec clang-format-15 -style=file -i {} \; \ No newline at end of file From 4b5f2c31fab15383097b12ecc22ab5f49af04dd6 Mon Sep 17 00:00:00 2001 From: "shuai.xu" Date: Fri, 7 Jun 2024 16:29:51 +0800 Subject: [PATCH 226/402] [GLUTEN-5910] [CH] add custom type to ASTLiteral (#5911) What changes were proposed in this pull request? This pr based on ClickHouse/ClickHouse#64562, Type of Literal DateTime32 will be lost during parsing plan of CH as it will be convert to Int16. (Fixes: #5910) How was this patch tested? This patch was tested manually. --- ...tenClickHouseTPCHSaltNullParquetSuite.scala | 18 ++++++++++++++++++ .../Parser/SerializedPlanParser.cpp | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index ee495457edee..84f3901397ea 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2612,5 +2612,23 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("drop table ineq_join_t2") } } + + test("GLUTEN-5910: Fix ASTLiteral type is lost in CH") { + spark.sql("create table test_tbl_5910_0(c_time bigint, type int) using parquet") + spark.sql("create table test_tbl_5910_1(type int) using parquet") + spark.sql("insert into test_tbl_5910_0 values(1717209159, 12)") + spark.sql("insert into test_tbl_5910_1 values(12)") + val select_sql = + """ + | select t1.cday, t2.type from ( + | select type, to_date(from_unixtime(c_time)) as cday from test_tbl_5910_0 ) t1 + | left join ( + | select type, "2024-06-01" as cday from test_tbl_5910_1 ) t2 + | on t1.cday = t2.cday and t1.type = t2.type + |""".stripMargin + compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) + spark.sql("drop table test_tbl_5910_0") + spark.sql("drop table test_tbl_5910_1") + } } // scalastyle:on line.size.limit diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 1c907035a5ce..5f2c9cc33150 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -1888,8 +1888,8 @@ ASTPtr ASTParser::parseArgumentToAST(const Names & names, const substrait::Expre case substrait::Expression::RexTypeCase::kLiteral: { DataTypePtr type; Field field; - std::tie(std::ignore, field) = SerializedPlanParser::parseLiteral(rel.literal()); - return std::make_shared(field); + std::tie(type, field) = SerializedPlanParser::parseLiteral(rel.literal()); + return std::make_shared(field, type); } case substrait::Expression::RexTypeCase::kSelection: { if (!rel.selection().has_direct_reference() || !rel.selection().direct_reference().has_struct_field()) From 1e4793cf172fb447122e406494d1d7121173ca79 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 7 Jun 2024 17:23:27 +0800 Subject: [PATCH 227/402] [VL] Daily Update Velox Version (2024_06_07) (#6007) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 584c9d40d5b9..d749f9ff2e71 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_05 +VELOX_BRANCH=2024_06_07 VELOX_HOME="" #Set on run gluten on HDFS From dc853f6af7ed569dce16d1f44f374642755c6207 Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Fri, 7 Jun 2024 15:23:24 +0530 Subject: [PATCH 228/402] [VL] Update to_utc_timestamp and from_utc_timestamp tests (#5358) --- .../utils/velox/VeloxTestSettings.scala | 8 -- .../spark/sql/GlutenDateFunctionsSuite.scala | 76 ------------------- .../utils/velox/VeloxTestSettings.scala | 8 -- .../spark/sql/GlutenDateFunctionsSuite.scala | 76 ------------------- .../utils/velox/VeloxTestSettings.scala | 8 -- .../spark/sql/GlutenDateFunctionsSuite.scala | 76 ------------------- .../utils/velox/VeloxTestSettings.scala | 8 -- .../spark/sql/GlutenDateFunctionsSuite.scala | 76 ------------------- 8 files changed, 336 deletions(-) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5762855c2cde..5df53953e4cc 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -265,14 +265,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") - // Replaced by another test. - .exclude("to_utc_timestamp with literal zone") - // Replaced by another test. - .exclude("to_utc_timestamp with column zone") - // Replaced by another test - .exclude("from_utc_timestamp with literal zone") - // Replaced by another test - .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDataFrameFunctionsSuite] // blocked by Velox-5768 .exclude("aggregate function - array for primitive type containing null") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index 154e67ae3ae0..8d1f7320dd42 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -248,80 +248,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra } } } - - testGluten("to_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - } - - testGluten("to_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris") - ).toDF("a", "b", "c") - checkAnswer( - df.select(to_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - } - - testGluten("from_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } - - testGluten("from_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles") - ).toDF("a", "b", "c") - checkAnswer( - df.select(from_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } } diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 898fc2b39583..79357e9e220d 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1070,14 +1070,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") - // Replaced by another test. - .exclude("to_utc_timestamp with literal zone") - // Replaced by another test. - .exclude("to_utc_timestamp with column zone") - // Replaced by another test - .exclude("from_utc_timestamp with literal zone") - // Replaced by another test - .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index fbbfdf45daa8..a946e6de4345 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra } } } - - testGluten("to_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - } - - testGluten("to_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris") - ).toDF("a", "b", "c") - checkAnswer( - df.select(to_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - } - - testGluten("from_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } - - testGluten("from_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles") - ).toDF("a", "b", "c") - checkAnswer( - df.select(from_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index bd437bbe8efb..c532355df6de 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1073,14 +1073,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") - // Replaced by another test. - .exclude("to_utc_timestamp with literal zone") - // Replaced by another test. - .exclude("to_utc_timestamp with column zone") - // Replaced by another test - .exclude("from_utc_timestamp with literal zone") - // Replaced by another test - .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index fbbfdf45daa8..a946e6de4345 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra } } } - - testGluten("to_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - } - - testGluten("to_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris") - ).toDF("a", "b", "c") - checkAnswer( - df.select(to_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - } - - testGluten("from_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } - - testGluten("from_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles") - ).toDF("a", "b", "c") - checkAnswer( - df.select(from_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index af8d0deadfc8..104c22dbe482 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1090,14 +1090,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") - // Replaced by another test. - .exclude("to_utc_timestamp with literal zone") - // Replaced by another test. - .exclude("to_utc_timestamp with column zone") - // Replaced by another test - .exclude("from_utc_timestamp with literal zone") - // Replaced by another test - .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index fbbfdf45daa8..a946e6de4345 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra } } } - - testGluten("to_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-25 07:00:00"))) - ) - } - - testGluten("to_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris") - ).toDF("a", "b", "c") - checkAnswer( - df.select(to_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - checkAnswer( - df.select(to_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 07:00:00")), - Row(Timestamp.valueOf("2015-07-24 22:00:00"))) - ) - } - - testGluten("from_utc_timestamp with literal zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00") - ).toDF("a", "b") - checkAnswer( - df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")), - Seq( - Row(Timestamp.valueOf("2015-07-23 17:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } - - testGluten("from_utc_timestamp with column zone") { - val df = Seq( - (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"), - (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles") - ).toDF("a", "b", "c") - checkAnswer( - df.select(from_utc_timestamp(col("a"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - checkAnswer( - df.select(from_utc_timestamp(col("b"), col("c"))), - Seq( - Row(Timestamp.valueOf("2015-07-24 02:00:00")), - Row(Timestamp.valueOf("2015-07-24 17:00:00"))) - ) - } } From d0dcc1e401ee1600f1e9716f5dffc1befb059634 Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Fri, 7 Jun 2024 18:02:01 +0800 Subject: [PATCH 229/402] [CH] Disable automatic switching of sort shuffle (#6015) What changes were proposed in this pull request? Disable automatic switching of sort shuffle How was this patch tested? unit tests (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) --- cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp index 559d9031862e..fd6f6fd81b5d 100644 --- a/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp +++ b/cpp-ch/local-engine/Shuffle/CachedShuffleWriter.cpp @@ -133,11 +133,11 @@ void CachedShuffleWriter::lazyInitPartitionWriter(Block & input_sample) if (partition_writer) return; - auto avg_row_size = input_sample.allocatedBytes() / input_sample.rows(); - auto overhead_memory = std::max(avg_row_size, input_sample.columns() * 16) * options.split_size * options.partition_num; - auto use_sort_shuffle = overhead_memory > options.spill_threshold * 0.5 || options.partition_num >= 300; +// auto avg_row_size = input_sample.allocatedBytes() / input_sample.rows(); +// auto overhead_memory = std::max(avg_row_size, input_sample.columns() * 16) * options.split_size * options.partition_num; +// auto use_sort_shuffle = overhead_memory > options.spill_threshold * 0.5 || options.partition_num >= 300; auto use_external_sort_shuffle = options.force_external_sort; - auto use_memory_sort_shuffle = options.force_mermory_sort || use_sort_shuffle; + auto use_memory_sort_shuffle = options.force_mermory_sort; sort_shuffle = use_memory_sort_shuffle || use_external_sort_shuffle; if (celeborn_client) { From 85e8619f124edbe73ded69efd2173b45c833c6c5 Mon Sep 17 00:00:00 2001 From: lgbo Date: Fri, 7 Jun 2024 18:57:52 +0800 Subject: [PATCH 230/402] [GLUTEN-5981][CH] Make the result be null when the queried field in get_json_object is null (#6001) [CH] Make the result be null when the queried field in get_json_object is null --- .../execution/GlutenClickhouseFunctionSuite.scala | 11 +++++++++++ .../Functions/SparkFunctionGetJsonObject.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala index 63e105e12a72..7e46a6989d6f 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala @@ -179,4 +179,15 @@ class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite { |""".stripMargin )(df => checkFallbackOperators(df, 0)) } + + test("GLUTEN-5981 null value from get_json_object") { + spark.sql("create table json_t1 (a string) using parquet") + spark.sql("insert into json_t1 values ('{\"a\":null}')") + runQueryAndCompare( + """ + |SELECT get_json_object(a, '$.a') is null from json_t1 + |""".stripMargin + )(df => checkFallbackOperators(df, 0)) + spark.sql("drop table json_t1") + } } diff --git a/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h b/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h index 6ba05e901791..1d9b9a0fd4c1 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h @@ -113,6 +113,8 @@ class GetJsonObjectImpl JSONStringSerializer serializer(*col_str); if (elements.size() == 1) [[likely]] { + if (elements[0].isNull()) + return false; nullable_col_str.getNullMapData().push_back(0); if (elements[0].isString()) { From 31c384f0d124293c28eea787260d6566ac51f9a4 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Sat, 8 Jun 2024 08:31:50 -0500 Subject: [PATCH 231/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240608) (#6023) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240608) * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/64798 * Fix UT --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 +- .../Functions/SparkFunctionFloor.h | 69 ++++++++++++----- .../Functions/SparkFunctionRoundHalfUp.h | 77 +++++++++---------- .../tests/gtest_parquet_columnindex.cpp | 2 +- 4 files changed, 91 insertions(+), 61 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index d98ce9f8852a..a5ca8d8dd0e1 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240606 -CH_COMMIT=fed1c01e169 +CH_BRANCH=rebase_ch/20240608 +CH_COMMIT=b5050282335 diff --git a/cpp-ch/local-engine/Functions/SparkFunctionFloor.h b/cpp-ch/local-engine/Functions/SparkFunctionFloor.h index b016c9afa407..ce33d11dbd8c 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionFloor.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionFloor.h @@ -14,13 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include +#pragma once + #include #include -#include #include -#include +#include +#include +#include using namespace DB; @@ -130,20 +131,29 @@ struct SparkFloatFloorImpl { private: static_assert(!is_decimal); - using Op = FloatRoundingComputation; - using Data = std::array; + template < + Vectorize vectorize = +#ifdef __SSE4_1__ + Vectorize::Yes +#else + Vectorize::No +#endif + > + using Op = FloatRoundingComputation; + using Data = std::array::data_count>; + public: static void apply(const PaddedPODArray & in, size_t scale, PaddedPODArray & out, PaddedPODArray & null_map) { - auto mm_scale = Op::prepare(scale); + auto mm_scale = Op<>::prepare(scale); const size_t data_count = std::tuple_size(); - const T* end_in = in.data() + in.size(); - const T* limit = in.data() + in.size() / data_count * data_count; - const T* __restrict p_in = in.data(); - T* __restrict p_out = out.data(); + const T * end_in = in.data() + in.size(); + const T * limit = in.data() + in.size() / data_count * data_count; + const T * __restrict p_in = in.data(); + T * __restrict p_out = out.data(); while (p_in < limit) { - Op::compute(p_in, mm_scale, p_out); + Op<>::compute(p_in, mm_scale, p_out); p_in += data_count; p_out += data_count; } @@ -154,7 +164,7 @@ struct SparkFloatFloorImpl Data tmp_dst; size_t tail_size_bytes = (end_in - p_in) * sizeof(*p_in); memcpy(&tmp_src, p_in, tail_size_bytes); - Op::compute(reinterpret_cast(&tmp_src), mm_scale, reinterpret_cast(&tmp_dst)); + Op<>::compute(reinterpret_cast(&tmp_src), mm_scale, reinterpret_cast(&tmp_dst)); memcpy(p_out, &tmp_dst, tail_size_bytes); } @@ -171,11 +181,31 @@ struct SparkFloatFloorImpl checkAndSetNullable(out[i], null_map[i]); } } - }; class SparkFunctionFloor : public DB::FunctionFloor { + static Scale getScaleArg(const ColumnsWithTypeAndName & arguments) + { + if (arguments.size() == 2) + { + const IColumn & scale_column = *arguments[1].column; + if (!isColumnConst(scale_column)) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Scale argument for rounding functions must be constant"); + + Field scale_field = assert_cast(scale_column).getField(); + if (scale_field.getType() != Field::Types::UInt64 && scale_field.getType() != Field::Types::Int64) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Scale argument for rounding functions must have integer type"); + + Int64 scale64 = scale_field.get(); + if (scale64 > std::numeric_limits::max() || scale64 < std::numeric_limits::min()) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale argument for rounding function is too large"); + + return scale64; + } + return 0; + } + public: static constexpr auto name = "sparkFloor"; static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared(); } @@ -183,17 +213,20 @@ class SparkFunctionFloor : public DB::FunctionFloor ~SparkFunctionFloor() override = default; String getName() const override { return name; } - DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes & arguments) const override + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + DB::DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { auto result_type = DB::FunctionFloor::getReturnTypeImpl(arguments); return makeNullable(result_type); } - DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows) const override + DB::ColumnPtr + executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows) const override { const ColumnWithTypeAndName & first_arg = arguments[0]; Scale scale_arg = getScaleArg(arguments); - switch(first_arg.type->getTypeId()) + switch (first_arg.type->getTypeId()) { case TypeIndex::Float32: return executeInternal(first_arg.column, scale_arg); @@ -206,7 +239,7 @@ class SparkFunctionFloor : public DB::FunctionFloor } } - template + template static ColumnPtr executeInternal(const ColumnPtr & col_arg, const Scale & scale_arg) { const auto * col = checkAndGetColumn>(col_arg.get()); diff --git a/cpp-ch/local-engine/Functions/SparkFunctionRoundHalfUp.h b/cpp-ch/local-engine/Functions/SparkFunctionRoundHalfUp.h index 47135aabd94f..441842d4e7e1 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionRoundHalfUp.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionRoundHalfUp.h @@ -18,6 +18,11 @@ #include +namespace DB::ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + namespace local_engine { using namespace DB; @@ -35,10 +40,11 @@ class BaseFloatRoundingHalfUpComputation static VectorType load(const ScalarType * in) { return _mm_loadu_ps(in); } static VectorType load1(const ScalarType in) { return _mm_load1_ps(&in); } - static void store(ScalarType * out, VectorType val) { _mm_storeu_ps(out, val);} + static void store(ScalarType * out, VectorType val) { _mm_storeu_ps(out, val); } static VectorType multiply(VectorType val, VectorType scale) { return _mm_mul_ps(val, scale); } static VectorType divide(VectorType val, VectorType scale) { return _mm_div_ps(val, scale); } - template static VectorType apply(VectorType val) + template + static VectorType apply(VectorType val) { ScalarType tempFloatsIn[data_count]; ScalarType tempFloatsOut[data_count]; @@ -49,10 +55,7 @@ class BaseFloatRoundingHalfUpComputation return load(tempFloatsOut); } - static VectorType prepare(size_t scale) - { - return load1(scale); - } + static VectorType prepare(size_t scale) { return load1(scale); } }; template <> @@ -65,10 +68,11 @@ class BaseFloatRoundingHalfUpComputation static VectorType load(const ScalarType * in) { return _mm_loadu_pd(in); } static VectorType load1(const ScalarType in) { return _mm_load1_pd(&in); } - static void store(ScalarType * out, VectorType val) { _mm_storeu_pd(out, val);} + static void store(ScalarType * out, VectorType val) { _mm_storeu_pd(out, val); } static VectorType multiply(VectorType val, VectorType scale) { return _mm_mul_pd(val, scale); } static VectorType divide(VectorType val, VectorType scale) { return _mm_div_pd(val, scale); } - template static VectorType apply(VectorType val) + template + static VectorType apply(VectorType val) { ScalarType tempFloatsIn[data_count]; ScalarType tempFloatsOut[data_count]; @@ -79,10 +83,7 @@ class BaseFloatRoundingHalfUpComputation return load(tempFloatsOut); } - static VectorType prepare(size_t scale) - { - return load1(scale); - } + static VectorType prepare(size_t scale) { return load1(scale); } }; @@ -135,11 +136,11 @@ struct FloatRoundingHalfUpImpl const size_t data_count = std::tuple_size(); - const T* end_in = in.data() + in.size(); - const T* limit = in.data() + in.size() / data_count * data_count; + const T * end_in = in.data() + in.size(); + const T * limit = in.data() + in.size() / data_count * data_count; - const T* __restrict p_in = in.data(); - T* __restrict p_out = out.data(); + const T * __restrict p_in = in.data(); + T * __restrict p_out = out.data(); while (p_in < limit) { @@ -169,9 +170,10 @@ template - using FunctionRoundingImpl = std::conditional_t, - FloatRoundingHalfUpImpl, - IntegerRoundingImpl>; + using FunctionRoundingImpl = std::conditional_t< + std::is_floating_point_v, + FloatRoundingHalfUpImpl, + IntegerRoundingImpl>; static ColumnPtr apply(const IColumn * col_general, Scale scale_arg) { @@ -233,10 +235,7 @@ class FunctionRoundingHalfUp : public IFunction static constexpr auto name = "roundHalfUp"; static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -246,14 +245,16 @@ class FunctionRoundingHalfUp : public IFunction DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if ((arguments.empty()) || (arguments.size() > 2)) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2.", - getName(), arguments.size()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2.", + getName(), + arguments.size()); for (const auto & type : arguments) if (!isNumber(type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", - arguments[0]->getName(), getName()); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); return arguments[0]; } @@ -267,13 +268,11 @@ class FunctionRoundingHalfUp : public IFunction throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Scale argument for rounding functions must be constant"); Field scale_field = assert_cast(scale_column).getField(); - if (scale_field.getType() != Field::Types::UInt64 - && scale_field.getType() != Field::Types::Int64) + if (scale_field.getType() != Field::Types::UInt64 && scale_field.getType() != Field::Types::Int64) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Scale argument for rounding functions must have integer type"); Int64 scale64 = scale_field.get(); - if (scale64 > std::numeric_limits::max() - || scale64 < std::numeric_limits::min()) + if (scale64 > std::numeric_limits::max() || scale64 < std::numeric_limits::min()) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale argument for rounding function is too large"); return scale64; @@ -305,26 +304,24 @@ class FunctionRoundingHalfUp : public IFunction }; if (!callOnIndexAndDataType(column.type->getTypeId(), call)) - { throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", column.name, getName()); - } return res; } - bool hasInformationAboutMonotonicity() const override - { - return true; - } + bool hasInformationAboutMonotonicity() const override { return true; } Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override { - return { .is_monotonic = true, .is_always_monotonic = true }; + return {.is_monotonic = true, .is_always_monotonic = true}; } }; -struct NameRoundHalfUp { static constexpr auto name = "roundHalfUp"; }; +struct NameRoundHalfUp +{ + static constexpr auto name = "roundHalfUp"; +}; using FunctionRoundHalfUp = FunctionRoundingHalfUp; diff --git a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp index 532244029b78..bdaa51f974c5 100644 --- a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp +++ b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp @@ -604,7 +604,7 @@ TEST(ColumnIndex, DecimalField) ASSERT_EQ(actual, expected); - /// Eexception test, only in relase release node + /// Exception test, only in release node #ifdef NDEBUG Field unsupport = DecimalField(Int256(300000000), 4); EXPECT_THROW(to_parquet.as(unsupport, desc), DB::Exception); From ec8d279d9f790fc41f4bf651b87df0943eb74f71 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Sun, 9 Jun 2024 18:38:19 +0800 Subject: [PATCH 232/402] [VL] Use mvn -ntp for all workflow jobs (#6025) --- .github/workflows/velox_docker.yml | 56 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index dc007c5760aa..ca065609163e 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -41,6 +41,8 @@ on: - 'cpp/core/**' - 'dev/**' +env: + MVN_CMD: 'mvn -ntp' concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -124,9 +126,9 @@ jobs: cd $GITHUB_WORKSPACE/ export JAVA_HOME=/usr/lib/jvm/${{ matrix.java }}-openjdk-amd64 echo "JAVA_HOME: $JAVA_HOME" - mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests + $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it - mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} \ + $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} \ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ @@ -191,9 +193,9 @@ jobs: run: | echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests + $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it - mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} + $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} - name: Run TPC-H / TPC-DS run: | echo "JAVA_HOME: $JAVA_HOME" @@ -254,9 +256,9 @@ jobs: - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests + $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it - mvn -ntp clean install -P${{ matrix.spark }} + $MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - name: TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory, memory isolation off run: | @@ -356,9 +358,9 @@ jobs: - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests + $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it - mvn -ntp clean install -P${{ matrix.spark }} + $MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - name: TPC-DS SF30.0 Parquet local spark3.2 random kill tasks run: | @@ -404,9 +406,9 @@ jobs: # - name: Build for Spark ${{ matrix.spark }} # run: | # cd $GITHUB_WORKSPACE/ - # mvn -ntp clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests + # $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests # cd $GITHUB_WORKSPACE/tools/gluten-it - # mvn -ntp clean install -P${{ matrix.spark }} + # $MVN_CMD clean install -P${{ matrix.spark }} # GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=h -s=30.0 --threads=12 # GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 # - name: TPC-H / TPC-DS SF30.0 Parquet local ${{ matrix.spark }} @@ -454,7 +456,7 @@ jobs: cd $GITHUB_WORKSPACE/ && \ export MAVEN_HOME=/usr/lib/maven && \ export PATH=${PATH}:${MAVEN_HOME}/bin && \ - mvn clean install -P${{ matrix.spark }} -Pbackends-velox -Puniffle -DskipTests + $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Puniffle -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with uniffle 0.8.0 run: | export MAVEN_HOME=/usr/lib/maven && \ @@ -466,7 +468,7 @@ jobs: sed -i '250d' ./server/src/main/java/org/apache/uniffle/server/ShuffleTaskManager.java && \ sed -i '228d' ./server/src/main/java/org/apache/uniffle/server/ShuffleTaskManager.java && \ sed -i '226d' ./server/src/main/java/org/apache/uniffle/server/ShuffleTaskManager.java && \ - mvn clean install -Phadoop2.8 -DskipTests + $MVN_CMD clean install -Phadoop2.8 -DskipTests cd /opt && \ wget -nv https://archive.apache.org/dist/incubator/uniffle/0.8.0/apache-uniffle-0.8.0-incubating-bin.tar.gz && \ tar xzf apache-uniffle-0.8.0-incubating-bin.tar.gz -C /opt/ && mv /opt/rss-0.8.0-hadoop2.8 /opt/uniffle && \ @@ -480,7 +482,7 @@ jobs: bash -c "echo -e 'rss.coordinator.shuffle.nodes.max 1\nrss.rpc.server.port 19999' > ./conf/coordinator.conf" && \ bash -c "echo -e 'rss.server.app.expired.withoutHeartbeat 7200000\nrss.server.heartbeat.delay 3000\nrss.rpc.server.port 19997\nrss.jetty.http.port 19996\nrss.server.netty.port 19995\nrss.storage.basePath /opt/uniffle/shuffle_data\nrss.storage.type MEMORY_LOCALFILE\nrss.coordinator.quorum localhost:19999\nrss.server.flush.thread.alive 10\nrss.server.single.buffer.flush.threshold 64m' > ./conf/server.conf" && \ bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh - cd $GITHUB_WORKSPACE/tools/gluten-it && mvn clean install -Pspark-3.2 -Puniffle && \ + cd $GITHUB_WORKSPACE/tools/gluten-it && $MVN_CMD clean install -Pspark-3.2 -Puniffle && \ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox-with-uniffle --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 @@ -511,7 +513,7 @@ jobs: - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ - mvn clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests + $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }} run: | export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 @@ -527,7 +529,7 @@ jobs: bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \ bash -c "echo -e 'celeborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf" && \ bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \ - cd $GITHUB_WORKSPACE/tools/gluten-it && mvn clean install -Pspark-3.2 -Pceleborn ${EXTRA_PROFILE} && \ + cd $GITHUB_WORKSPACE/tools/gluten-it && $MVN_CMD clean install -Pspark-3.2 -Pceleborn ${EXTRA_PROFILE} && \ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox-with-celeborn --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1 && \ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ @@ -653,8 +655,8 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn -ntp test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ + $MVN_CMD test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -715,7 +717,7 @@ jobs: - name: Build and run unit test for Spark 3.2.2 (slow tests) run: | cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark33: needs: build-native-lib-centos-8 @@ -782,8 +784,8 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn -ntp test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ + $MVN_CMD test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -840,7 +842,7 @@ jobs: - name: Build and Run unit test for Spark 3.3.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark34: needs: build-native-lib-centos-8 @@ -907,8 +909,8 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn -ntp test -Pspark-3.4 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ + $MVN_CMD test -Pspark-3.4 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -965,7 +967,7 @@ jobs: - name: Build and Run unit test for Spark 3.4.2 (slow tests) run: | cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark35: needs: build-native-lib-centos-8 @@ -1032,8 +1034,8 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - mvn -ntp clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn -ntp test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ + $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -1095,4 +1097,4 @@ jobs: - name: Build and Run unit test for Spark 3.5.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - mvn -ntp clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ No newline at end of file + $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ No newline at end of file From 805ec69a0d7e142dfb337ba6c7e7dd9d8484efe8 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Sun, 9 Jun 2024 19:33:04 +0800 Subject: [PATCH 233/402] [VL] Daily Update Velox Version (2024_06_09) (#6028) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index d749f9ff2e71..736f5ee20aaa 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_07 +VELOX_BRANCH=2024_06_08 VELOX_HOME="" #Set on run gluten on HDFS From df627bee60bd597ff3fb1067479ceda8069f8902 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 11 Jun 2024 09:13:39 +0800 Subject: [PATCH 234/402] [CI][VL] Re-enable native benchmark test (#6020) --- .github/workflows/velox_docker.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index ca065609163e..6c1be4344c71 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -581,6 +581,10 @@ jobs: with: name: udf-example-lib-centos-8-${{github.sha}} path: ./cpp/build/velox/udf/examples/ + - uses: actions/upload-artifact@v2 + with: + name: benchmark-centos-8-${{github.sha}} + path: ./cpp/build/velox/benchmarks/ - uses: actions/upload-artifact@v2 with: name: arrow-jars-centos-8-${{github.sha}} @@ -604,6 +608,11 @@ jobs: with: name: udf-example-lib-centos-8-${{github.sha}} path: ./cpp/build/velox/udf/examples/ + - name: Download Benchmark + uses: actions/download-artifact@v2 + with: + name: benchmark-centos-8-${{github.sha}} + path: ./cpp/build/velox/benchmarks/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -663,11 +672,12 @@ jobs: with: name: golden-files-spark32 path: /tmp/tpch-approved-plan/** - # - name: Gluten CPP Benchmark Test - # run: | - # # This test depends on example.json generated by the above mvn test. - # cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ - # ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 + - name: Gluten CPP Benchmark Test + run: | + # This test depends on example.json generated by the above mvn test. + cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ + sudo chmod +x ./generic_benchmark && \ + ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 run-spark-test-spark32-slow: needs: build-native-lib-centos-8 @@ -1097,4 +1107,4 @@ jobs: - name: Build and Run unit test for Spark 3.5.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ No newline at end of file + $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest From 13babf369d4a5f5dc9833bea7ec22bfa682f8ffe Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Tue, 11 Jun 2024 07:19:57 +0530 Subject: [PATCH 235/402] [VL] Pass file size and modification time in split (#6029) --- .../clickhouse/CHIteratorApi.scala | 5 +++- .../backendsapi/velox/VeloxIteratorApi.scala | 23 +++++++++++++++++-- cpp/velox/compute/VeloxPlanConverter.cc | 3 +++ cpp/velox/compute/WholeStageResultIterator.cc | 8 +++++-- cpp/velox/substrait/SubstraitToVeloxPlan.h | 5 ++++ .../substrait/rel/LocalFilesBuilder.java | 4 ++++ .../gluten/substrait/rel/LocalFilesNode.java | 18 +++++++++++++++ .../substrait/proto/substrait/algebra.proto | 7 ++++++ .../substrait/rel/IcebergLocalFilesNode.java | 2 ++ .../apache/gluten/sql/shims/SparkShims.scala | 3 +++ .../sql/shims/spark32/Spark32Shims.scala | 5 ++++ .../sql/shims/spark33/Spark33Shims.scala | 5 ++++ .../sql/shims/spark34/Spark34Shims.scala | 5 ++++ .../sql/shims/spark35/Spark35Shims.scala | 5 ++++ 14 files changed, 93 insertions(+), 5 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 63f7eeb798f3..1221710bce6b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -131,10 +131,13 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { paths, starts, lengths, + new JArrayList[JLong](), + new JArrayList[JLong](), partitionColumns, new JArrayList[JMap[String, String]](), fileFormat, - preferredLocations.toList.asJava) + preferredLocations.toList.asJava + ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition: $partition.") } diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 5f9b5afa9976..b20eccafb625 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -56,7 +56,14 @@ class VeloxIteratorApi extends IteratorApi with Logging { metadataColumnNames: Seq[String]): SplitInfo = { partition match { case f: FilePartition => - val (paths, starts, lengths, partitionColumns, metadataColumns) = + val ( + paths, + starts, + lengths, + fileSizes, + modificationTimes, + partitionColumns, + metadataColumns) = constructSplitInfo(partitionSchema, f.files, metadataColumnNames) val preferredLocations = SoftAffinity.getFilePartitionLocations(f) @@ -65,6 +72,8 @@ class VeloxIteratorApi extends IteratorApi with Logging { paths, starts, lengths, + fileSizes, + modificationTimes, partitionColumns, metadataColumns, fileFormat, @@ -100,6 +109,8 @@ class VeloxIteratorApi extends IteratorApi with Logging { val paths = new JArrayList[String]() val starts = new JArrayList[JLong] val lengths = new JArrayList[JLong]() + val fileSizes = new JArrayList[JLong]() + val modificationTimes = new JArrayList[JLong]() val partitionColumns = new JArrayList[JMap[String, String]] var metadataColumns = new JArrayList[JMap[String, String]] files.foreach { @@ -111,6 +122,14 @@ class VeloxIteratorApi extends IteratorApi with Logging { .decode(file.filePath.toString, StandardCharsets.UTF_8.name())) starts.add(JLong.valueOf(file.start)) lengths.add(JLong.valueOf(file.length)) + val (fileSize, modificationTime) = + SparkShimLoader.getSparkShims.getFileSizeAndModificationTime(file) + (fileSize, modificationTime) match { + case (Some(size), Some(time)) => + fileSizes.add(JLong.valueOf(size)) + modificationTimes.add(JLong.valueOf(time)) + case _ => // Do nothing + } val metadataColumn = SparkShimLoader.getSparkShims.generateMetadataColumns(file, metadataColumnNames) metadataColumns.add(metadataColumn) @@ -138,7 +157,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { } partitionColumns.add(partitionColumn) } - (paths, starts, lengths, partitionColumns, metadataColumns) + (paths, starts, lengths, fileSizes, modificationTimes, partitionColumns, metadataColumns) } override def injectWriteFilesTempPath(path: String): Unit = { diff --git a/cpp/velox/compute/VeloxPlanConverter.cc b/cpp/velox/compute/VeloxPlanConverter.cc index ed42cb15a51e..bcd03b110afd 100644 --- a/cpp/velox/compute/VeloxPlanConverter.cc +++ b/cpp/velox/compute/VeloxPlanConverter.cc @@ -60,6 +60,7 @@ std::shared_ptr parseScanSplitInfo( splitInfo->starts.reserve(fileList.size()); splitInfo->lengths.reserve(fileList.size()); splitInfo->partitionColumns.reserve(fileList.size()); + splitInfo->properties.reserve(fileList.size()); splitInfo->metadataColumns.reserve(fileList.size()); for (const auto& file : fileList) { // Expect all Partitions share the same index. @@ -80,6 +81,8 @@ std::shared_ptr parseScanSplitInfo( splitInfo->paths.emplace_back(file.uri_file()); splitInfo->starts.emplace_back(file.start()); splitInfo->lengths.emplace_back(file.length()); + facebook::velox::FileProperties fileProps = {file.properties().filesize(), file.properties().modificationtime()}; + splitInfo->properties.emplace_back(fileProps); switch (file.file_format_case()) { case SubstraitFileFormatCase::kOrc: splitInfo->format = dwio::common::FileFormat::ORC; diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index f719c119c3e0..867d347cdc64 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -109,6 +109,7 @@ WholeStageResultIterator::WholeStageResultIterator( const auto& paths = scanInfo->paths; const auto& starts = scanInfo->starts; const auto& lengths = scanInfo->lengths; + const auto& properties = scanInfo->properties; const auto& format = scanInfo->format; const auto& partitionColumns = scanInfo->partitionColumns; const auto& metadataColumns = scanInfo->metadataColumns; @@ -135,7 +136,9 @@ WholeStageResultIterator::WholeStageResultIterator( std::nullopt, customSplitInfo, nullptr, - deleteFiles); + deleteFiles, + std::unordered_map(), + properties[idx]); } else { split = std::make_shared( kHiveConnectorId, @@ -149,7 +152,8 @@ WholeStageResultIterator::WholeStageResultIterator( nullptr, std::unordered_map(), 0, - metadataColumn); + metadataColumn, + properties[idx]); } connectorSplits.emplace_back(split); } diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.h b/cpp/velox/substrait/SubstraitToVeloxPlan.h index 1bda6435eaee..567ebb215078 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.h +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.h @@ -19,6 +19,7 @@ #include "SubstraitToVeloxExpr.h" #include "TypeUtils.h" +#include "velox/connectors/hive/FileProperties.h" #include "velox/connectors/hive/TableHandle.h" #include "velox/core/PlanNode.h" #include "velox/dwio/common/Options.h" @@ -51,6 +52,9 @@ struct SplitInfo { /// The file format of the files to be scanned. dwio::common::FileFormat format; + /// The file sizes and modification times of the files to be scanned. + std::vector> properties; + /// Make SplitInfo polymorphic virtual ~SplitInfo() = default; }; @@ -111,6 +115,7 @@ class SubstraitToVeloxPlanConverter { /// Index: the index of the partition this item belongs to. /// Starts: the start positions in byte to read from the items. /// Lengths: the lengths in byte to read from the items. + /// FileProperties: the file sizes and modification times of the files to be scanned. core::PlanNodePtr toVeloxPlan(const ::substrait::ReadRel& sRead); core::PlanNodePtr constructValueStreamNode(const ::substrait::ReadRel& sRead, int32_t streamIdx); diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java index 94acc83367f5..7e085f81f4e6 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java @@ -27,6 +27,8 @@ public static LocalFilesNode makeLocalFiles( List paths, List starts, List lengths, + List fileSizes, + List modificationTimes, List> partitionColumns, List> metadataColumns, LocalFilesNode.ReadFileFormat fileFormat, @@ -36,6 +38,8 @@ public static LocalFilesNode makeLocalFiles( paths, starts, lengths, + fileSizes, + modificationTimes, partitionColumns, metadataColumns, fileFormat, diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java index cbcda72dd03a..fa9f3d51612b 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java @@ -34,6 +34,8 @@ public class LocalFilesNode implements SplitInfo { private final List paths = new ArrayList<>(); private final List starts = new ArrayList<>(); private final List lengths = new ArrayList<>(); + private final List fileSizes = new ArrayList<>(); + private final List modificationTimes = new ArrayList<>(); private final List> partitionColumns = new ArrayList<>(); private final List> metadataColumns = new ArrayList<>(); private final List preferredLocations = new ArrayList<>(); @@ -60,6 +62,8 @@ public enum ReadFileFormat { List paths, List starts, List lengths, + List fileSizes, + List modificationTimes, List> partitionColumns, List> metadataColumns, ReadFileFormat fileFormat, @@ -68,6 +72,8 @@ public enum ReadFileFormat { this.paths.addAll(paths); this.starts.addAll(starts); this.lengths.addAll(lengths); + this.fileSizes.addAll(fileSizes); + this.modificationTimes.addAll(modificationTimes); this.fileFormat = fileFormat; this.partitionColumns.addAll(partitionColumns); this.metadataColumns.addAll(metadataColumns); @@ -153,6 +159,18 @@ public ReadRel.LocalFiles toProtobuf() { } fileBuilder.setLength(lengths.get(i)); fileBuilder.setStart(starts.get(i)); + + if (!fileSizes.isEmpty() + && !modificationTimes.isEmpty() + && fileSizes.size() == modificationTimes.size() + && fileSizes.size() == paths.size()) { + ReadRel.LocalFiles.FileOrFiles.fileProperties.Builder filePropsBuilder = + ReadRel.LocalFiles.FileOrFiles.fileProperties.newBuilder(); + filePropsBuilder.setFileSize(fileSizes.get(i)); + filePropsBuilder.setModificationTime(modificationTimes.get(i)); + fileBuilder.setProperties(filePropsBuilder.build()); + } + if (!metadataColumns.isEmpty()) { Map metadataColumn = metadataColumns.get(i); if (!metadataColumn.isEmpty()) { diff --git a/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto b/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto index 266aba4b0157..877493439f95 100644 --- a/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto +++ b/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto @@ -198,6 +198,13 @@ message ReadRel { string value = 2; } repeated metadataColumn metadata_columns = 19; + + // File properties contained in split + message fileProperties { + int64 fileSize = 1; + int64 modificationTime = 2; + } + fileProperties properties = 20; } } } diff --git a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java b/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java index 7d065f105a43..ba6b0ac4a029 100644 --- a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java +++ b/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java @@ -42,6 +42,8 @@ public class IcebergLocalFilesNode extends LocalFilesNode { paths, starts, lengths, + new ArrayList<>(), + new ArrayList<>(), partitionColumns, new ArrayList<>(), fileFormat, diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala index d6acc8c27b29..8bbc6d3d18d4 100644 --- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala @@ -207,6 +207,9 @@ trait SparkShims { def attributesFromStruct(structType: StructType): Seq[Attribute] + // Spark 3.3 and later only have file size and modification time in PartitionedFile + def getFileSizeAndModificationTime(file: PartitionedFile): (Option[Long], Option[Long]) + def generateMetadataColumns( file: PartitionedFile, metadataColumnNames: Seq[String] = Seq.empty): JMap[String, String] diff --git a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala index 29fddc697b07..f24aef66a1cb 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala @@ -217,6 +217,11 @@ class Spark32Shims extends SparkShims { } } + override def getFileSizeAndModificationTime( + file: PartitionedFile): (Option[Long], Option[Long]) = { + (None, None) + } + override def generateMetadataColumns( file: PartitionedFile, metadataColumnNames: Seq[String]): JMap[String, String] = diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala index 7c6ce644dc74..68fc4ad0dc1a 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala @@ -202,6 +202,11 @@ class Spark33Shims extends SparkShims { case other => other } + override def getFileSizeAndModificationTime( + file: PartitionedFile): (Option[Long], Option[Long]) = { + (Some(file.fileSize), Some(file.modificationTime)) + } + override def generateMetadataColumns( file: PartitionedFile, metadataColumnNames: Seq[String]): JMap[String, String] = { diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index f2c2482949b7..7d9fc389b7cb 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -208,6 +208,11 @@ class Spark34Shims extends SparkShims { case other => other } + override def getFileSizeAndModificationTime( + file: PartitionedFile): (Option[Long], Option[Long]) = { + (Some(file.fileSize), Some(file.modificationTime)) + } + override def generateMetadataColumns( file: PartitionedFile, metadataColumnNames: Seq[String]): JMap[String, String] = { diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index e0835c3069d2..54cea6993d13 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -206,6 +206,11 @@ class Spark35Shims extends SparkShims { case other => other } + override def getFileSizeAndModificationTime( + file: PartitionedFile): (Option[Long], Option[Long]) = { + (Some(file.fileSize), Some(file.modificationTime)) + } + override def generateMetadataColumns( file: PartitionedFile, metadataColumnNames: Seq[String]): JMap[String, String] = { From ec3e92ec841ef71490c8c41c64647c60888e3885 Mon Sep 17 00:00:00 2001 From: YunDa <56379080+XinShuoWang@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:51:09 +0800 Subject: [PATCH 236/402] [VL] Optimize the performance of hash based shuffle by accumulating batches --- cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc | 16 +++++++++++++++- cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h | 9 +++++++++ cpp/velox/shuffle/VeloxShuffleWriter.h | 4 ++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc index 741ca8ab9b40..cc648cf7fdd0 100644 --- a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc @@ -303,7 +303,17 @@ arrow::Status VeloxHashBasedShuffleWriter::write(std::shared_ptr numRows -= length; } while (numRows); } else { - RETURN_NOT_OK(partitioningAndDoSplit(std::move(rv), memLimit)); + if (accumulateRows_ + rv->size() < 8192) { + accumulateRows_ += rv->size(); + initAccumulateDataset(rv); + accumulateDataset_->append(rv.get()); + } else { + initAccumulateDataset(rv); + accumulateDataset_->append(rv.get()); + RETURN_NOT_OK(partitioningAndDoSplit(std::move(accumulateDataset_), memLimit)); + accumulateDataset_ = nullptr; + accumulateRows_ = 0; + } } } return arrow::Status::OK(); @@ -329,6 +339,10 @@ arrow::Status VeloxHashBasedShuffleWriter::partitioningAndDoSplit(facebook::velo } arrow::Status VeloxHashBasedShuffleWriter::stop() { + if (accumulateDataset_ != nullptr) { + RETURN_NOT_OK(partitioningAndDoSplit(std::move(accumulateDataset_), kMinMemLimit)); + accumulateRows_ = 0; + } if (options_.partitioning != Partitioning::kSingle) { for (auto pid = 0; pid < numPartitions_; ++pid) { RETURN_NOT_OK(evictPartitionBuffers(pid, false)); diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h index a11f84e952a6..142c7978bdc9 100644 --- a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h @@ -303,6 +303,15 @@ class VeloxHashBasedShuffleWriter : public VeloxShuffleWriter { arrow::Status partitioningAndDoSplit(facebook::velox::RowVectorPtr rv, int64_t memLimit); + void initAccumulateDataset(facebook::velox::RowVectorPtr& rv) { + if (accumulateDataset_) { + return; + } + std::vector children(rv->children().size(), nullptr); + accumulateDataset_ = + std::make_shared(veloxPool_.get(), rv->type(), nullptr, 0, std::move(children)); + } + BinaryArrayResizeState binaryArrayResizeState_{}; bool hasComplexType_ = false; diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.h b/cpp/velox/shuffle/VeloxShuffleWriter.h index 104b87616291..2855831c51ae 100644 --- a/cpp/velox/shuffle/VeloxShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxShuffleWriter.h @@ -124,6 +124,10 @@ class VeloxShuffleWriter : public ShuffleWriter { int32_t maxBatchSize_{0}; + uint32_t accumulateRows_{0}; + + facebook::velox::RowVectorPtr accumulateDataset_; + enum EvictState { kEvictable, kUnevictable }; // stat From f9ab77236844b76b961ca5609423d0d0c91102d0 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 11 Jun 2024 10:20:55 +0800 Subject: [PATCH 237/402] [VL] RAS: Validate against all offloaded plan nodes to decide whether to do this offload (#6017) --- .../extension/columnar/enumerated/RasOffload.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala index 6af89dc057aa..8091127da0bf 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala @@ -105,13 +105,13 @@ object RasOffload { validator.validate(from) match { case Validator.Passed => val offloaded = base.offload(from) - offloaded match { - case t: GlutenPlan if !t.doValidate().isValid => - // 4. If native validation fails on the offloaded node, return the - // original one. - from - case other => - other + val offloadedNodes = offloaded.collect[GlutenPlan] { case t: GlutenPlan => t } + if (offloadedNodes.exists(!_.doValidate().isValid)) { + // 4. If native validation fails on the offloaded node, return the + // original one. + from + } else { + offloaded } case Validator.Failed(reason) => from From 70e5832d5fcf5451108db4bb1a8eb397e6c56447 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:47:22 +0800 Subject: [PATCH 238/402] [VL] Daily Update Velox Version (2024_06_11) (#6034) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 736f5ee20aaa..4965dbac66cd 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_08 +VELOX_BRANCH=2024_06_11 VELOX_HOME="" #Set on run gluten on HDFS From 1d0bb52248097cba721e850e83c0963971dbc81c Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 11 Jun 2024 13:57:46 +0800 Subject: [PATCH 239/402] [VL] Provide options to combine small batches before sending to shuffle (#6009) --- .../clickhouse/CHSparkPlanExecApi.scala | 9 +- .../gluten/utils/VeloxBatchAppender.java | 41 +++++++ .../utils/VeloxBatchAppenderJniWrapper.java | 41 +++++++ .../backendsapi/velox/VeloxIteratorApi.scala | 4 +- .../velox/VeloxSparkPlanExecApi.scala | 44 ++++---- .../execution/VeloxAppendBatchesExec.scala | 105 ++++++++++++++++++ .../gluten/execution/TestOperator.scala | 26 ++++- cpp/core/jni/JniCommon.cc | 58 ++++++++++ cpp/core/jni/JniCommon.h | 41 +++++++ cpp/core/jni/JniWrapper.cc | 88 --------------- cpp/velox/CMakeLists.txt | 1 + cpp/velox/jni/VeloxJniWrapper.cc | 18 +++ .../shuffle/VeloxHashBasedShuffleWriter.cc | 16 +-- .../shuffle/VeloxHashBasedShuffleWriter.h | 9 -- cpp/velox/shuffle/VeloxShuffleWriter.h | 4 - cpp/velox/utils/VeloxBatchAppender.cc | 59 ++++++++++ cpp/velox/utils/VeloxBatchAppender.h | 41 +++++++ .../gluten/backendsapi/SparkPlanExecApi.scala | 2 +- .../columnar/OffloadSingleNode.scala | 28 ++--- .../org/apache/gluten/utils/Iterators.scala | 38 ++++++- .../vectorized/ColumnarBatchOutIterator.java | 3 +- .../org/apache/gluten/GlutenConfig.scala | 26 +++++ 22 files changed, 532 insertions(+), 170 deletions(-) create mode 100644 backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java create mode 100644 backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java create mode 100644 backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala create mode 100644 cpp/velox/utils/VeloxBatchAppender.cc create mode 100644 cpp/velox/utils/VeloxBatchAppender.h diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index bdbdfed0d0d0..a8a05c40f1cd 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -271,13 +271,10 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { } } - override def genColumnarShuffleExchange( - shuffle: ShuffleExchangeExec, - child: SparkPlan): SparkPlan = { + override def genColumnarShuffleExchange(shuffle: ShuffleExchangeExec): SparkPlan = { + val child = shuffle.child if ( - BackendsApiManager.getSettings.supportShuffleWithProject( - shuffle.outputPartitioning, - shuffle.child) + BackendsApiManager.getSettings.supportShuffleWithProject(shuffle.outputPartitioning, child) ) { val (projectColumnNumber, newPartitioning, newChild) = addProjectionForShuffleExchange(shuffle) diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java new file mode 100644 index 000000000000..1bf34b5ce3b6 --- /dev/null +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.utils; + +import org.apache.gluten.exec.Runtime; +import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.memory.nmm.NativeMemoryManager; +import org.apache.gluten.memory.nmm.NativeMemoryManagers; +import org.apache.gluten.vectorized.ColumnarBatchInIterator; +import org.apache.gluten.vectorized.ColumnarBatchOutIterator; + +import org.apache.spark.sql.vectorized.ColumnarBatch; + +import java.util.Iterator; + +public final class VeloxBatchAppender { + public static ColumnarBatchOutIterator create( + int minOutputBatchSize, Iterator in) { + final Runtime runtime = Runtimes.contextInstance(); + final NativeMemoryManager nmm = NativeMemoryManagers.contextInstance("VeloxBatchAppender"); + long outHandle = + VeloxBatchAppenderJniWrapper.forRuntime(runtime) + .create( + nmm.getNativeInstanceHandle(), minOutputBatchSize, new ColumnarBatchInIterator(in)); + return new ColumnarBatchOutIterator(runtime, outHandle, nmm); + } +} diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java new file mode 100644 index 000000000000..9e2531951ccc --- /dev/null +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.utils; + +import org.apache.gluten.exec.Runtime; +import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.vectorized.ColumnarBatchInIterator; + +public class VeloxBatchAppenderJniWrapper implements RuntimeAware { + private final Runtime runtime; + + private VeloxBatchAppenderJniWrapper(Runtime runtime) { + this.runtime = runtime; + } + + public static VeloxBatchAppenderJniWrapper forRuntime(Runtime runtime) { + return new VeloxBatchAppenderJniWrapper(runtime); + } + + @Override + public long handle() { + return runtime.getHandle(); + } + + public native long create( + long memoryManagerHandle, int minOutputBatchSize, ColumnarBatchInIterator itr); +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index b20eccafb625..459a7886ea23 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -203,7 +203,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { resIter.close() } .recyclePayload(batch => batch.close()) - .addToPipelineTime(pipelineTime) + .collectLifeMillis(millis => pipelineTime += millis) .asInterruptible(context) .create() } @@ -246,7 +246,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { nativeResultIterator.close() } .recyclePayload(batch => batch.close()) - .addToPipelineTime(pipelineTime) + .collectLifeMillis(millis => pipelineTime += millis) .create() } // scalastyle:on argcount diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index f8af80a9b44d..66ca8660a50c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -320,9 +320,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { aggregateAttributes: Seq[Attribute]): HashAggregateExecPullOutBaseHelper = HashAggregateExecPullOutHelper(aggregateExpressions, aggregateAttributes) - override def genColumnarShuffleExchange( - shuffle: ShuffleExchangeExec, - newChild: SparkPlan): SparkPlan = { + override def genColumnarShuffleExchange(shuffle: ShuffleExchangeExec): SparkPlan = { def allowHashOnMap[T](f: => T): T = { val originalAllowHash = SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE) try { @@ -333,20 +331,28 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { } } + def maybeAddAppendBatchesExec(plan: SparkPlan): SparkPlan = { + if (GlutenConfig.getConf.veloxCoalesceBatchesBeforeShuffle) { + VeloxAppendBatchesExec(plan, GlutenConfig.getConf.veloxMinBatchSizeForShuffle) + } else { + plan + } + } + + val child = shuffle.child + shuffle.outputPartitioning match { case HashPartitioning(exprs, _) => val hashExpr = new Murmur3Hash(exprs) - val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ newChild.output - val projectTransformer = ProjectExecTransformer(projectList, newChild) + val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ child.output + val projectTransformer = ProjectExecTransformer(projectList, child) val validationResult = projectTransformer.doValidate() if (validationResult.isValid) { - ColumnarShuffleExchangeExec( - shuffle, - projectTransformer, - projectTransformer.output.drop(1)) + val newChild = maybeAddAppendBatchesExec(projectTransformer) + ColumnarShuffleExchangeExec(shuffle, newChild, newChild.output.drop(1)) } else { TransformHints.tagNotTransformable(shuffle, validationResult) - shuffle.withNewChildren(newChild :: Nil) + shuffle.withNewChildren(child :: Nil) } case RoundRobinPartitioning(num) if SQLConf.get.sortBeforeRepartition && num > 1 => // scalastyle:off line.size.limit @@ -357,19 +363,20 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { allowHashOnMap { // Velox hash expression does not support null type and we also do not need to sort // null type since the value always be null. - val columnsForHash = newChild.output.filterNot(_.dataType == NullType) + val columnsForHash = child.output.filterNot(_.dataType == NullType) if (columnsForHash.isEmpty) { + val newChild = maybeAddAppendBatchesExec(child) ColumnarShuffleExchangeExec(shuffle, newChild, newChild.output) } else { val hashExpr = new Murmur3Hash(columnsForHash) - val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ newChild.output - val projectTransformer = ProjectExecTransformer(projectList, newChild) + val projectList = Seq(Alias(hashExpr, "hash_partition_key")()) ++ child.output + val projectTransformer = ProjectExecTransformer(projectList, child) val projectBeforeSortValidationResult = projectTransformer.doValidate() // Make sure we support offload hash expression val projectBeforeSort = if (projectBeforeSortValidationResult.isValid) { projectTransformer } else { - val project = ProjectExec(projectList, newChild) + val project = ProjectExec(projectList, child) TransformHints.tagNotTransformable(project, projectBeforeSortValidationResult) project } @@ -380,17 +387,16 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { ProjectExecTransformer(projectList.drop(1), sortByHashCode) val validationResult = dropSortColumnTransformer.doValidate() if (validationResult.isValid) { - ColumnarShuffleExchangeExec( - shuffle, - dropSortColumnTransformer, - dropSortColumnTransformer.output) + val newChild = maybeAddAppendBatchesExec(dropSortColumnTransformer) + ColumnarShuffleExchangeExec(shuffle, newChild, newChild.output) } else { TransformHints.tagNotTransformable(shuffle, validationResult) - shuffle.withNewChildren(newChild :: Nil) + shuffle.withNewChildren(child :: Nil) } } } case _ => + val newChild = maybeAddAppendBatchesExec(child) ColumnarShuffleExchangeExec(shuffle, newChild, null) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala new file mode 100644 index 000000000000..8c2834574204 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.utils.{Iterators, VeloxBatchAppender} + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.physical.Partitioning +import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode} +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.vectorized.ColumnarBatch + +import java.util.concurrent.atomic.AtomicLong + +import scala.collection.JavaConverters._ + +/** + * An operator to coalesce input batches by appending the later batches to the one that comes + * earlier. + */ +case class VeloxAppendBatchesExec(override val child: SparkPlan, minOutputBatchSize: Int) + extends GlutenPlan + with UnaryExecNode { + + override lazy val metrics: Map[String, SQLMetric] = Map( + "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"), + "numInputBatches" -> SQLMetrics.createMetric(sparkContext, "number of input batches"), + "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), + "numOutputBatches" -> SQLMetrics.createMetric(sparkContext, "number of output batches"), + "appendTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to append batches") + ) + + override def supportsColumnar: Boolean = true + override protected def doExecute(): RDD[InternalRow] = throw new UnsupportedOperationException() + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val numInputRows = longMetric("numInputRows") + val numInputBatches = longMetric("numInputBatches") + val numOutputRows = longMetric("numOutputRows") + val numOutputBatches = longMetric("numOutputBatches") + val appendTime = longMetric("appendTime") + + child.executeColumnar().mapPartitions { + in => + // Append millis = Out millis - In millis. + val appendMillis = new AtomicLong(0L) + + val appender = VeloxBatchAppender.create( + minOutputBatchSize, + Iterators + .wrap(in) + .collectReadMillis(inMillis => appendMillis.getAndAdd(-inMillis)) + .create() + .map { + inBatch => + numInputRows += inBatch.numRows() + numInputBatches += 1 + inBatch + } + .asJava + ) + + val out = Iterators + .wrap(appender.asScala) + .collectReadMillis(outMillis => appendMillis.getAndAdd(outMillis)) + .recyclePayload(_.close()) + .recycleIterator { + appender.close() + appendTime += appendMillis.get() + } + .create() + .map { + outBatch => + numOutputRows += outBatch.numRows() + numOutputBatches += 1 + outBatch + } + + out + } + } + + override def output: Seq[Attribute] = child.output + override def outputPartitioning: Partitioning = child.outputPartitioning + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = + copy(child = newChild) +} diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index cd1f21a0a31c..ae8d64a09937 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -24,6 +24,7 @@ import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf import org.apache.spark.sql.{AnalysisException, DataFrame, Row} import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf @@ -33,7 +34,7 @@ import java.util.concurrent.TimeUnit import scala.collection.JavaConverters -class TestOperator extends VeloxWholeStageTransformerSuite { +class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPlanHelper { protected val rootPath: String = getClass.getResource("/").getPath override protected val resourcePath: String = "/tpch-data-parquet-velox" @@ -703,6 +704,29 @@ class TestOperator extends VeloxWholeStageTransformerSuite { } } + test("combine small batches before shuffle") { + val minBatchSize = 15 + withSQLConf( + "spark.gluten.sql.columnar.backend.velox.coalesceBatchesBeforeShuffle" -> "true", + "spark.gluten.sql.columnar.maxBatchSize" -> "2", + "spark.gluten.sql.columnar.backend.velox.minBatchSizeForShuffle" -> s"$minBatchSize" + ) { + val df = runQueryAndCompare( + "select l_orderkey, sum(l_partkey) as sum from lineitem " + + "where l_orderkey < 100 group by l_orderkey") { _ => } + checkLengthAndPlan(df, 27) + val ops = collect(df.queryExecution.executedPlan) { case p: VeloxAppendBatchesExec => p } + assert(ops.size == 1) + val op = ops.head + assert(op.minOutputBatchSize == minBatchSize) + val metrics = op.metrics + assert(metrics("numInputRows").value == 27) + assert(metrics("numInputBatches").value == 14) + assert(metrics("numOutputRows").value == 27) + assert(metrics("numOutputBatches").value == 2) + } + } + test("test OneRowRelation") { val df = sql("SELECT 1") checkAnswer(df, Row(1)) diff --git a/cpp/core/jni/JniCommon.cc b/cpp/core/jni/JniCommon.cc index 328a7b7722f9..08c5cb1d40cb 100644 --- a/cpp/core/jni/JniCommon.cc +++ b/cpp/core/jni/JniCommon.cc @@ -65,3 +65,61 @@ gluten::Runtime* gluten::getRuntime(JNIEnv* env, jobject runtimeAware) { GLUTEN_CHECK(ctx != nullptr, "FATAL: resource instance should not be null."); return ctx; } + +std::unique_ptr gluten::makeJniColumnarBatchIterator( + JNIEnv* env, + jobject jColumnarBatchItr, + gluten::Runtime* runtime, + std::shared_ptr writer) { + return std::make_unique(env, jColumnarBatchItr, runtime, writer); +} + +gluten::JniColumnarBatchIterator::JniColumnarBatchIterator( + JNIEnv* env, + jobject jColumnarBatchItr, + gluten::Runtime* runtime, + std::shared_ptr writer) + : runtime_(runtime), writer_(writer) { + // IMPORTANT: DO NOT USE LOCAL REF IN DIFFERENT THREAD + if (env->GetJavaVM(&vm_) != JNI_OK) { + std::string errorMessage = "Unable to get JavaVM instance"; + throw gluten::GlutenException(errorMessage); + } + serializedColumnarBatchIteratorClass_ = + createGlobalClassReferenceOrError(env, "Lorg/apache/gluten/vectorized/ColumnarBatchInIterator;"); + serializedColumnarBatchIteratorHasNext_ = + getMethodIdOrError(env, serializedColumnarBatchIteratorClass_, "hasNext", "()Z"); + serializedColumnarBatchIteratorNext_ = getMethodIdOrError(env, serializedColumnarBatchIteratorClass_, "next", "()J"); + jColumnarBatchItr_ = env->NewGlobalRef(jColumnarBatchItr); +} + +gluten::JniColumnarBatchIterator::~JniColumnarBatchIterator() { + JNIEnv* env; + attachCurrentThreadAsDaemonOrThrow(vm_, &env); + env->DeleteGlobalRef(jColumnarBatchItr_); + env->DeleteGlobalRef(serializedColumnarBatchIteratorClass_); + vm_->DetachCurrentThread(); +} + +std::shared_ptr gluten::JniColumnarBatchIterator::next() { + JNIEnv* env; + attachCurrentThreadAsDaemonOrThrow(vm_, &env); + if (!env->CallBooleanMethod(jColumnarBatchItr_, serializedColumnarBatchIteratorHasNext_)) { + checkException(env); + return nullptr; // stream ended + } + + checkException(env); + jlong handle = env->CallLongMethod(jColumnarBatchItr_, serializedColumnarBatchIteratorNext_); + checkException(env); + auto batch = runtime_->objectStore()->retrieve(handle); + if (writer_ != nullptr) { + // save snapshot of the batch to file + std::shared_ptr schema = batch->exportArrowSchema(); + std::shared_ptr array = batch->exportArrowArray(); + auto rb = gluten::arrowGetOrThrow(arrow::ImportRecordBatch(array.get(), schema.get())); + GLUTEN_THROW_NOT_OK(writer_->initWriter(*(rb->schema().get()))); + GLUTEN_THROW_NOT_OK(writer_->writeInBatches(rb)); + } + return batch; +} diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index 5858a70e9a77..bc5cf84f6ff4 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -28,6 +28,7 @@ #include "memory/AllocationListener.h" #include "shuffle/rss/RssClient.h" #include "utils/Compression.h" +#include "utils/ResourceMap.h" #include "utils/exception.h" static jint jniVersion = JNI_VERSION_1_8; @@ -119,6 +120,12 @@ static inline void attachCurrentThreadAsDaemonOrThrow(JavaVM* vm, JNIEnv** out) } } +template +static T* jniCastOrThrow(gluten::ResourceHandle handle) { + auto instance = reinterpret_cast(handle); + GLUTEN_CHECK(instance != nullptr, "FATAL: resource instance should not be null."); + return instance; +} namespace gluten { class JniCommonState { @@ -251,6 +258,40 @@ DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kLong, jlongArray, Long) DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kFloat, jfloatArray, Float) DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kDouble, jdoubleArray, Double) +class JniColumnarBatchIterator : public ColumnarBatchIterator { + public: + explicit JniColumnarBatchIterator( + JNIEnv* env, + jobject jColumnarBatchItr, + Runtime* runtime, + std::shared_ptr writer); + + // singleton + JniColumnarBatchIterator(const JniColumnarBatchIterator&) = delete; + JniColumnarBatchIterator(JniColumnarBatchIterator&&) = delete; + JniColumnarBatchIterator& operator=(const JniColumnarBatchIterator&) = delete; + JniColumnarBatchIterator& operator=(JniColumnarBatchIterator&&) = delete; + + virtual ~JniColumnarBatchIterator(); + + std::shared_ptr next() override; + + private: + JavaVM* vm_; + jobject jColumnarBatchItr_; + Runtime* runtime_; + std::shared_ptr writer_; + + jclass serializedColumnarBatchIteratorClass_; + jmethodID serializedColumnarBatchIteratorHasNext_; + jmethodID serializedColumnarBatchIteratorNext_; +}; + +std::unique_ptr makeJniColumnarBatchIterator( + JNIEnv* env, + jobject jColumnarBatchItr, + Runtime* runtime, + std::shared_ptr writer); } // namespace gluten // TODO: Move the static functions to namespace gluten diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index db498f43adbf..4e069ec7a6d6 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -58,13 +58,8 @@ static jmethodID splitResultConstructor; static jclass columnarBatchSerializeResultClass; static jmethodID columnarBatchSerializeResultConstructor; -static jclass serializedColumnarBatchIteratorClass; static jclass metricsBuilderClass; static jmethodID metricsBuilderConstructor; - -static jmethodID serializedColumnarBatchIteratorHasNext; -static jmethodID serializedColumnarBatchIteratorNext; - static jclass nativeColumnarToRowInfoClass; static jmethodID nativeColumnarToRowInfoConstructor; @@ -147,80 +142,6 @@ class JavaInputStreamAdaptor final : public arrow::io::InputStream { bool closed_ = false; }; -class JniColumnarBatchIterator : public ColumnarBatchIterator { - public: - explicit JniColumnarBatchIterator( - JNIEnv* env, - jobject jColumnarBatchItr, - Runtime* runtime, - std::shared_ptr writer) - : runtime_(runtime), writer_(writer) { - // IMPORTANT: DO NOT USE LOCAL REF IN DIFFERENT THREAD - if (env->GetJavaVM(&vm_) != JNI_OK) { - std::string errorMessage = "Unable to get JavaVM instance"; - throw gluten::GlutenException(errorMessage); - } - jColumnarBatchItr_ = env->NewGlobalRef(jColumnarBatchItr); - } - - // singleton - JniColumnarBatchIterator(const JniColumnarBatchIterator&) = delete; - JniColumnarBatchIterator(JniColumnarBatchIterator&&) = delete; - JniColumnarBatchIterator& operator=(const JniColumnarBatchIterator&) = delete; - JniColumnarBatchIterator& operator=(JniColumnarBatchIterator&&) = delete; - - virtual ~JniColumnarBatchIterator() { - JNIEnv* env; - attachCurrentThreadAsDaemonOrThrow(vm_, &env); - env->DeleteGlobalRef(jColumnarBatchItr_); - vm_->DetachCurrentThread(); - } - - std::shared_ptr next() override { - JNIEnv* env; - attachCurrentThreadAsDaemonOrThrow(vm_, &env); - if (!env->CallBooleanMethod(jColumnarBatchItr_, serializedColumnarBatchIteratorHasNext)) { - checkException(env); - return nullptr; // stream ended - } - - checkException(env); - jlong handle = env->CallLongMethod(jColumnarBatchItr_, serializedColumnarBatchIteratorNext); - checkException(env); - auto batch = runtime_->objectStore()->retrieve(handle); - if (writer_ != nullptr) { - // save snapshot of the batch to file - std::shared_ptr schema = batch->exportArrowSchema(); - std::shared_ptr array = batch->exportArrowArray(); - auto rb = gluten::arrowGetOrThrow(arrow::ImportRecordBatch(array.get(), schema.get())); - GLUTEN_THROW_NOT_OK(writer_->initWriter(*(rb->schema().get()))); - GLUTEN_THROW_NOT_OK(writer_->writeInBatches(rb)); - } - return batch; - } - - private: - JavaVM* vm_; - jobject jColumnarBatchItr_; - Runtime* runtime_; - std::shared_ptr writer_; -}; - -std::unique_ptr makeJniColumnarBatchIterator( - JNIEnv* env, - jobject jColumnarBatchItr, - Runtime* runtime, - std::shared_ptr writer) { - return std::make_unique(env, jColumnarBatchItr, runtime, writer); -} - -template -T* jniCastOrThrow(ResourceHandle handle) { - auto instance = reinterpret_cast(handle); - GLUTEN_CHECK(instance != nullptr, "FATAL: resource instance should not be null."); - return instance; -} - #ifdef __cplusplus extern "C" { #endif @@ -253,14 +174,6 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { metricsBuilderConstructor = getMethodIdOrError( env, metricsBuilderClass, "", "([J[J[J[J[J[J[J[J[J[JJ[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J)V"); - serializedColumnarBatchIteratorClass = - createGlobalClassReferenceOrError(env, "Lorg/apache/gluten/vectorized/ColumnarBatchInIterator;"); - - serializedColumnarBatchIteratorHasNext = - getMethodIdOrError(env, serializedColumnarBatchIteratorClass, "hasNext", "()Z"); - - serializedColumnarBatchIteratorNext = getMethodIdOrError(env, serializedColumnarBatchIteratorClass, "next", "()J"); - nativeColumnarToRowInfoClass = createGlobalClassReferenceOrError(env, "Lorg/apache/gluten/vectorized/NativeColumnarToRowInfo;"); nativeColumnarToRowInfoConstructor = getMethodIdOrError(env, nativeColumnarToRowInfoClass, "", "([I[IJ)V"); @@ -293,7 +206,6 @@ void JNI_OnUnload(JavaVM* vm, void* reserved) { env->DeleteGlobalRef(jniByteInputStreamClass); env->DeleteGlobalRef(splitResultClass); env->DeleteGlobalRef(columnarBatchSerializeResultClass); - env->DeleteGlobalRef(serializedColumnarBatchIteratorClass); env->DeleteGlobalRef(nativeColumnarToRowInfoClass); env->DeleteGlobalRef(byteArrayClass); env->DeleteGlobalRef(shuffleReaderMetricsClass); diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 05ecf9635eb0..34cc9001cf38 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -327,6 +327,7 @@ set(VELOX_SRCS utils/VeloxArrowUtils.cc utils/ConfigExtractor.cc utils/Common.cc + utils/VeloxBatchAppender.cc ) if (ENABLE_HDFS) diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index 9da7355d1b3a..3b52eaa86b2f 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -30,6 +30,7 @@ #include "jni/JniFileSystem.h" #include "memory/VeloxMemoryManager.h" #include "substrait/SubstraitToVeloxPlanValidator.h" +#include "utils/VeloxBatchAppender.h" #include "velox/common/base/BloomFilter.h" #include @@ -246,6 +247,23 @@ JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWra JNI_METHOD_END(nullptr) } +JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBatchAppenderJniWrapper_create( // NOLINT + JNIEnv* env, + jobject wrapper, + jlong memoryManagerHandle, + jint minOutputBatchSize, + jobject jIter) { + JNI_METHOD_START + auto ctx = gluten::getRuntime(env, wrapper); + auto memoryManager = jniCastOrThrow(memoryManagerHandle); + auto pool = gluten::VeloxRuntime::getLeafVeloxPool(memoryManager); + auto iter = gluten::makeJniColumnarBatchIterator(env, jIter, ctx, nullptr); + auto appender = std::make_shared( + std::make_unique(pool.get(), minOutputBatchSize, std::move(iter))); + return ctx->objectStore()->save(appender); + JNI_METHOD_END(gluten::kInvalidResourceHandle) +} + #ifdef __cplusplus } #endif diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc index cc648cf7fdd0..741ca8ab9b40 100644 --- a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc @@ -303,17 +303,7 @@ arrow::Status VeloxHashBasedShuffleWriter::write(std::shared_ptr numRows -= length; } while (numRows); } else { - if (accumulateRows_ + rv->size() < 8192) { - accumulateRows_ += rv->size(); - initAccumulateDataset(rv); - accumulateDataset_->append(rv.get()); - } else { - initAccumulateDataset(rv); - accumulateDataset_->append(rv.get()); - RETURN_NOT_OK(partitioningAndDoSplit(std::move(accumulateDataset_), memLimit)); - accumulateDataset_ = nullptr; - accumulateRows_ = 0; - } + RETURN_NOT_OK(partitioningAndDoSplit(std::move(rv), memLimit)); } } return arrow::Status::OK(); @@ -339,10 +329,6 @@ arrow::Status VeloxHashBasedShuffleWriter::partitioningAndDoSplit(facebook::velo } arrow::Status VeloxHashBasedShuffleWriter::stop() { - if (accumulateDataset_ != nullptr) { - RETURN_NOT_OK(partitioningAndDoSplit(std::move(accumulateDataset_), kMinMemLimit)); - accumulateRows_ = 0; - } if (options_.partitioning != Partitioning::kSingle) { for (auto pid = 0; pid < numPartitions_; ++pid) { RETURN_NOT_OK(evictPartitionBuffers(pid, false)); diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h index 142c7978bdc9..a11f84e952a6 100644 --- a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.h @@ -303,15 +303,6 @@ class VeloxHashBasedShuffleWriter : public VeloxShuffleWriter { arrow::Status partitioningAndDoSplit(facebook::velox::RowVectorPtr rv, int64_t memLimit); - void initAccumulateDataset(facebook::velox::RowVectorPtr& rv) { - if (accumulateDataset_) { - return; - } - std::vector children(rv->children().size(), nullptr); - accumulateDataset_ = - std::make_shared(veloxPool_.get(), rv->type(), nullptr, 0, std::move(children)); - } - BinaryArrayResizeState binaryArrayResizeState_{}; bool hasComplexType_ = false; diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.h b/cpp/velox/shuffle/VeloxShuffleWriter.h index 2855831c51ae..104b87616291 100644 --- a/cpp/velox/shuffle/VeloxShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxShuffleWriter.h @@ -124,10 +124,6 @@ class VeloxShuffleWriter : public ShuffleWriter { int32_t maxBatchSize_{0}; - uint32_t accumulateRows_{0}; - - facebook::velox::RowVectorPtr accumulateDataset_; - enum EvictState { kEvictable, kUnevictable }; // stat diff --git a/cpp/velox/utils/VeloxBatchAppender.cc b/cpp/velox/utils/VeloxBatchAppender.cc new file mode 100644 index 000000000000..8fa1ade217e0 --- /dev/null +++ b/cpp/velox/utils/VeloxBatchAppender.cc @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "VeloxBatchAppender.h" + +namespace gluten { + +gluten::VeloxBatchAppender::VeloxBatchAppender( + facebook::velox::memory::MemoryPool* pool, + int32_t minOutputBatchSize, + std::unique_ptr in) + : pool_(pool), minOutputBatchSize_(minOutputBatchSize), in_(std::move(in)) {} + +std::shared_ptr VeloxBatchAppender::next() { + auto cb = in_->next(); + if (cb == nullptr) { + // Input iterator was drained. + return nullptr; + } + if (cb->numRows() >= minOutputBatchSize_) { + // Fast flush path. + return cb; + } + + auto vb = VeloxColumnarBatch::from(pool_, cb); + auto rv = vb->getRowVector(); + auto buffer = facebook::velox::RowVector::createEmpty(rv->type(), pool_); + buffer->append(rv.get()); + + for (auto nextCb = in_->next(); nextCb != nullptr; nextCb = in_->next()) { + auto nextVb = VeloxColumnarBatch::from(pool_, nextCb); + auto nextRv = nextVb->getRowVector(); + buffer->append(nextRv.get()); + if (buffer->size() >= minOutputBatchSize_) { + // Buffer is full. + break; + } + } + return std::make_shared(buffer); +} + +int64_t VeloxBatchAppender::spillFixedSize(int64_t size) { + return in_->spillFixedSize(size); +} +} // namespace gluten diff --git a/cpp/velox/utils/VeloxBatchAppender.h b/cpp/velox/utils/VeloxBatchAppender.h new file mode 100644 index 000000000000..3698381d0add --- /dev/null +++ b/cpp/velox/utils/VeloxBatchAppender.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "memory/ColumnarBatchIterator.h" +#include "memory/VeloxColumnarBatch.h" +#include "utils/exception.h" +#include "velox/common/memory/MemoryPool.h" +#include "velox/vector/ComplexVector.h" + +namespace gluten { +class VeloxBatchAppender : public ColumnarBatchIterator { + public: + VeloxBatchAppender( + facebook::velox::memory::MemoryPool* pool, + int32_t minOutputBatchSize, + std::unique_ptr in); + + std::shared_ptr next() override; + + int64_t spillFixedSize(int64_t size) override; + + private: + facebook::velox::memory::MemoryPool* pool_; + const int32_t minOutputBatchSize_; + std::unique_ptr in_; +}; +} // namespace gluten diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 8a086f896ba4..8a1baae51092 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -101,7 +101,7 @@ trait SparkPlanExecApi { aggregateExpressions: Seq[AggregateExpression], aggregateAttributes: Seq[Attribute]): HashAggregateExecPullOutBaseHelper - def genColumnarShuffleExchange(shuffle: ShuffleExchangeExec, newChild: SparkPlan): SparkPlan + def genColumnarShuffleExchange(shuffle: ShuffleExchangeExec): SparkPlan /** Generate ShuffledHashJoinExecTransformer. */ def genShuffledHashJoinExecTransformer( diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 39cc8ad2e2e6..6e4d37f633eb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -101,23 +101,17 @@ case class OffloadAggregate() extends OffloadSingleNode with LogLevelUtil { // Exchange transformation. case class OffloadExchange() extends OffloadSingleNode with LogLevelUtil { override def offload(plan: SparkPlan): SparkPlan = plan match { - case plan if TransformHints.isNotTransformable(plan) => - plan - case plan: ShuffleExchangeExec => - logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - val child = plan.child - if ( - (child.supportsColumnar || GlutenConfig.getConf.enablePreferColumnar) && - BackendsApiManager.getSettings.supportColumnarShuffleExec() - ) { - BackendsApiManager.getSparkPlanExecApiInstance.genColumnarShuffleExchange(plan, child) - } else { - plan.withNewChildren(Seq(child)) - } - case plan: BroadcastExchangeExec => - val child = plan.child - logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - ColumnarBroadcastExchangeExec(plan.mode, child) + case p if TransformHints.isNotTransformable(p) => + p + case s: ShuffleExchangeExec + if (s.child.supportsColumnar || GlutenConfig.getConf.enablePreferColumnar) && + BackendsApiManager.getSettings.supportColumnarShuffleExec() => + logDebug(s"Columnar Processing for ${s.getClass} is currently supported.") + BackendsApiManager.getSparkPlanExecApiInstance.genColumnarShuffleExchange(s) + case b: BroadcastExchangeExec => + val child = b.child + logDebug(s"Columnar Processing for ${b.getClass} is currently supported.") + ColumnarBroadcastExchangeExec(b.mode, child) case other => other } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala index 81ff2dc0b177..1e3681355d6c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala @@ -17,7 +17,6 @@ package org.apache.gluten.utils import org.apache.spark.{InterruptibleIterator, TaskContext} -import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.util.TaskResources import java.util.concurrent.TimeUnit @@ -85,12 +84,12 @@ private class IteratorCompleter[A](in: Iterator[A])(completionCallback: => Unit) } } -private class PipelineTimeAccumulator[A](in: Iterator[A], pipelineTime: SQLMetric) +private class LifeTimeAccumulator[A](in: Iterator[A], onCollected: Long => Unit) extends Iterator[A] { private val closed = new AtomicBoolean(false) private val startTime = System.nanoTime() - TaskResources.addRecycler("Iterators#PipelineTimeAccumulator", 100) { + TaskResources.addRecycler("Iterators#LifeTimeAccumulator", 100) { tryFinish() } @@ -111,9 +110,31 @@ private class PipelineTimeAccumulator[A](in: Iterator[A], pipelineTime: SQLMetri if (!closed.compareAndSet(false, true)) { return } - pipelineTime += TimeUnit.NANOSECONDS.toMillis( + val lifeTime = TimeUnit.NANOSECONDS.toMillis( System.nanoTime() - startTime ) + onCollected(lifeTime) + } +} + +private class ReadTimeAccumulator[A](in: Iterator[A], onAdded: Long => Unit) extends Iterator[A] { + + override def hasNext: Boolean = { + val prev = System.nanoTime() + val out = in.hasNext + val after = System.nanoTime() + val duration = TimeUnit.NANOSECONDS.toMillis(after - prev) + onAdded(duration) + out + } + + override def next(): A = { + val prev = System.nanoTime() + val out = in.next() + val after = System.nanoTime() + val duration = TimeUnit.NANOSECONDS.toMillis(after - prev) + onAdded(duration) + out } } @@ -171,8 +192,13 @@ class WrapperBuilder[A](in: Iterator[A]) { // FIXME how to make the ctor compani this } - def addToPipelineTime(pipelineTime: SQLMetric): WrapperBuilder[A] = { - wrapped = new PipelineTimeAccumulator[A](wrapped, pipelineTime) + def collectLifeMillis(onCollected: Long => Unit): WrapperBuilder[A] = { + wrapped = new LifeTimeAccumulator[A](wrapped, onCollected) + this + } + + def collectReadMillis(onAdded: Long => Unit): WrapperBuilder[A] = { + wrapped = new ReadTimeAccumulator[A](wrapped, onAdded) this } diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index 37de9894392c..3a2a741bef0b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -31,8 +31,7 @@ public class ColumnarBatchOutIterator extends GeneralOutIterator implements Runt private final long iterHandle; private final NativeMemoryManager nmm; - public ColumnarBatchOutIterator(Runtime runtime, long iterHandle, NativeMemoryManager nmm) - throws IOException { + public ColumnarBatchOutIterator(Runtime runtime, long iterHandle, NativeMemoryManager nmm) { super(); this.runtime = runtime; this.iterHandle = iterHandle; diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index d76e698dcf2a..2376a1f39c1e 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -187,6 +187,7 @@ class GlutenConfig(conf: SQLConf) extends Logging { def columnarShuffleCompressionThreshold: Int = conf.getConf(COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD) + // FIXME: Not clear: MIN or MAX ? def maxBatchSize: Int = conf.getConf(COLUMNAR_MAX_BATCH_SIZE) def shuffleWriterBufferSize: Int = conf @@ -295,6 +296,14 @@ class GlutenConfig(conf: SQLConf) extends Logging { def veloxBloomFilterMaxNumBits: Long = conf.getConf(COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS) + def veloxCoalesceBatchesBeforeShuffle: Boolean = + conf.getConf(COLUMNAR_VELOX_COALESCE_BATCHES_BEFORE_SHUFFLE) + + def veloxMinBatchSizeForShuffle: Int = + conf + .getConf(COLUMNAR_VELOX_MIN_BATCH_SIZE_FOR_SHUFFLE) + .getOrElse(conf.getConf(COLUMNAR_MAX_BATCH_SIZE)) + def chColumnarShufflePreferSpill: Boolean = conf.getConf(COLUMNAR_CH_SHUFFLE_PREFER_SPILL_ENABLED) def chColumnarShuffleSpillThreshold: Long = { @@ -1395,6 +1404,23 @@ object GlutenConfig { .checkValue(_ > 0, "must be a positive number") .createWithDefault(10000) + val COLUMNAR_VELOX_COALESCE_BATCHES_BEFORE_SHUFFLE = + buildConf("spark.gluten.sql.columnar.backend.velox.coalesceBatchesBeforeShuffle") + .internal() + .doc(s"If true, combine small columnar batches together before sending to shuffle. " + + s"The default minimum output batch size is equal to $GLUTEN_MAX_BATCH_SIZE_KEY") + .booleanConf + .createWithDefault(false) + + val COLUMNAR_VELOX_MIN_BATCH_SIZE_FOR_SHUFFLE = + buildConf("spark.gluten.sql.columnar.backend.velox.minBatchSizeForShuffle") + .internal() + .doc(s"The minimum batch size for shuffle. If the batch size is smaller than this value, " + + s"it will be combined with other batches before sending to shuffle. Only functions when " + + s"${COLUMNAR_VELOX_COALESCE_BATCHES_BEFORE_SHUFFLE.key} is set to true.") + .intConf + .createOptional + val COLUMNAR_CH_SHUFFLE_PREFER_SPILL_ENABLED = buildConf("spark.gluten.sql.columnar.backend.ch.shuffle.preferSpill") .internal() From d3bd3d775e1d0b66aac92bf511df80fcfa3b2dec Mon Sep 17 00:00:00 2001 From: Xuedong Luan Date: Tue, 11 Jun 2024 14:20:45 +0800 Subject: [PATCH 240/402] [VL] Add gluten iceberg jar to bundle package (#6008) * Add iceberg to package * update doc * fix comment --- docs/get-started/Velox.md | 3 +-- gluten-iceberg/pom.xml | 4 ---- package/pom.xml | 10 ++++++++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index a24c99fda302..964744bd5eb5 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -303,8 +303,7 @@ First of all, compile gluten-iceberg module by a `iceberg` profile, as follows: mvn clean package -Pbackends-velox -Pspark-3.3 -Piceberg -DskipTests ``` -Then, put the additional `gluten-iceberg-XX-SNAPSHOT.jar` to the class path (usually it's `$SPARK_HOME/jars`). -The gluten-iceberg jar is in `gluten-iceberg/target` directory. +Once built successfully, iceberg features will be included in gluten-velox-bundle-X jar. Then you can query iceberg table by gluten/velox without scan's fallback. After the two steps, you can query iceberg table by gluten/velox without scan's fallback. diff --git a/gluten-iceberg/pom.xml b/gluten-iceberg/pom.xml index 727077f26797..69630b8cf0ec 100644 --- a/gluten-iceberg/pom.xml +++ b/gluten-iceberg/pom.xml @@ -63,10 +63,6 @@ test-jar test - - org.apache.spark - spark-core_${scala.binary.version} - org.apache.spark spark-core_${scala.binary.version} diff --git a/package/pom.xml b/package/pom.xml index db4056a7e109..ab87e14805ff 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -83,6 +83,16 @@ + + iceberg + + + org.apache.gluten + gluten-iceberg + ${project.version} + + + From b37a6e42681fa23b94337e7646153e5640e474d7 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:56:32 +0800 Subject: [PATCH 241/402] [GLUTEN-5827][CH]support utc timestamp transfrom (#5828) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #5827) support to_utc_timestamp/from_utc_timestamp function; convert timezone like '+08:00,-08:00' to GMT+8 , GMT-8 while this config set in spark.sql.session.timezone or in to_utc_timestamp / from_utc_timestamp's parameters How was this patch tested? spark ut --- .../gluten/utils/CHExpressionUtil.scala | 13 +++- cpp-ch/local-engine/Common/CHUtil.cpp | 18 ++++-- cpp-ch/local-engine/Common/CHUtil.h | 1 + .../Parser/SerializedPlanParser.h | 2 +- .../fromUtcTimestamp.cpp | 35 +++++++++++ .../scalar_function_parser/toUtcTimestamp.cpp | 35 +++++++++++ .../utcTimestampTransform.h | 61 +++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 2 - .../clickhouse/ClickHouseTestSettings.scala | 2 - .../clickhouse/ClickHouseTestSettings.scala | 2 - .../clickhouse/ClickHouseTestSettings.scala | 2 - 11 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/fromUtcTimestamp.cpp create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/toUtcTimestamp.cpp create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/utcTimestampTransform.h diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index 94d2895943d4..d47523c0f6ac 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -44,6 +44,15 @@ case class SequenceValidator() extends FunctionValidator { } } +case class UtcTimestampValidator() extends FunctionValidator { + override def doValidate(expr: Expression): Boolean = expr match { + // CH backend doest not support non-const timezone parameter + case t: ToUTCTimestamp => t.children(1).isInstanceOf[Literal] + case f: FromUTCTimestamp => f.children(1).isInstanceOf[Literal] + case _ => false + } +} + case class UnixTimeStampValidator() extends FunctionValidator { final val DATE_TYPE = "date" @@ -194,8 +203,8 @@ object CHExpressionUtil { REGR_SLOPE -> DefaultValidator(), REGR_INTERCEPT -> DefaultValidator(), REGR_SXY -> DefaultValidator(), - TO_UTC_TIMESTAMP -> DefaultValidator(), - FROM_UTC_TIMESTAMP -> DefaultValidator(), + TO_UTC_TIMESTAMP -> UtcTimestampValidator(), + FROM_UTC_TIMESTAMP -> UtcTimestampValidator(), UNIX_MILLIS -> DefaultValidator(), UNIX_MICROS -> DefaultValidator(), TIMESTAMP_MILLIS -> DefaultValidator(), diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 62b42f981168..fa6124cf011f 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -586,7 +586,7 @@ void BackendInitializerUtil::initEnvs(DB::Context::ConfigurationPtr config) if (config->has("timezone")) { const std::string config_timezone = config->getString("timezone"); - const String mapped_timezone = DateLUT::mappingForJavaTimezone(config_timezone); + const String mapped_timezone = DateTimeUtil::convertTimeZone(config_timezone); if (0 != setenv("TZ", mapped_timezone.data(), 1)) // NOLINT(concurrency-mt-unsafe) // ok if not called concurrently with other setenv/getenv throw Poco::Exception("Cannot setenv TZ variable"); @@ -659,11 +659,7 @@ void BackendInitializerUtil::initSettings(std::map & b } else if (key == SPARK_SESSION_TIME_ZONE) { - String time_zone_val = value; - /// Convert timezone ID like '+8:00' to GMT+8:00 - if (value.starts_with("+") || value.starts_with("-")) - time_zone_val = "GMT" + value; - time_zone_val = DateLUT::mappingForJavaTimezone(time_zone_val); + String time_zone_val = DateTimeUtil::convertTimeZone(value); settings.set("session_timezone", time_zone_val); LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{} value:{}", "session_timezone", time_zone_val); } @@ -937,6 +933,16 @@ Int64 DateTimeUtil::currentTimeMillis() return timeInMilliseconds(std::chrono::system_clock::now()); } +String DateTimeUtil::convertTimeZone(const String & time_zone) +{ + String res = time_zone; + /// Convert timezone ID like '+08:00' to GMT+8:00 + if (time_zone.starts_with("+") || time_zone.starts_with("-")) + res = "GMT" + time_zone; + res = DateLUT::mappingForJavaTimezone(res); + return res; +} + UInt64 MemoryUtil::getCurrentMemoryUsage(size_t depth) { Int64 current_memory_usage = 0; diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 2ef3c6ef99df..50de9461f4de 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -225,6 +225,7 @@ class DateTimeUtil { public: static Int64 currentTimeMillis(); + static String convertTimeZone(const String & time_zone); }; class MemoryUtil diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index d0a16ec71a47..c79598c59923 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -57,7 +57,7 @@ static const std::map SCALAR_FUNCTIONS {"get_timestamp", "parseDateTimeInJodaSyntaxOrNull"}, // for spark function: to_date/to_timestamp {"quarter", "toQuarter"}, {"to_unix_timestamp", "parseDateTimeInJodaSyntaxOrNull"}, - // {"unix_timestamp", "toUnixTimestamp"}, + //{"unix_timestamp", "toUnixTimestamp"}, {"date_format", "formatDateTimeInJodaSyntax"}, {"timestamp_add", "timestamp_add"}, diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/fromUtcTimestamp.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/fromUtcTimestamp.cpp new file mode 100644 index 000000000000..8d23231055c3 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/fromUtcTimestamp.cpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace local_engine +{ + +class FunctionParserFromUtcTimestamp : public FunctionParserUtcTimestampTransform +{ +public: + explicit FunctionParserFromUtcTimestamp(SerializedPlanParser * plan_parser_) : FunctionParserUtcTimestampTransform(plan_parser_) { } + ~FunctionParserFromUtcTimestamp() = default; + + static constexpr auto name = "from_utc_timestamp"; + String getCHFunctionName(const substrait::Expression_ScalarFunction &) const override { return "from_utc_timestamp"; } + String getName() const override { return "from_utc_timestamp"; } +}; + +static FunctionParserRegister fromUtcTimestamp; +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/toUtcTimestamp.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/toUtcTimestamp.cpp new file mode 100644 index 000000000000..4b04942bab31 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/toUtcTimestamp.cpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +namespace local_engine +{ + +class FunctionParserToUtcTimestamp : public FunctionParserUtcTimestampTransform +{ +public: + explicit FunctionParserToUtcTimestamp(SerializedPlanParser * plan_parser_) : FunctionParserUtcTimestampTransform(plan_parser_) { } + ~FunctionParserToUtcTimestamp() = default; + + static constexpr auto name = "to_utc_timestamp"; + String getCHFunctionName(const substrait::Expression_ScalarFunction &) const override { return "to_utc_timestamp"; } + String getName() const override { return "to_utc_timestamp"; } +}; + +static FunctionParserRegister toUtcTimestamp; +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/utcTimestampTransform.h b/cpp-ch/local-engine/Parser/scalar_function_parser/utcTimestampTransform.h new file mode 100644 index 000000000000..87ea19024169 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/utcTimestampTransform.h @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} +} + +namespace local_engine +{ + +class FunctionParserUtcTimestampTransform : public FunctionParser +{ +public: + explicit FunctionParserUtcTimestampTransform(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } + ~FunctionParserUtcTimestampTransform() override = default; + + const ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const override + { + /// Convert timezone value to clickhouse backend supported, i.e. GMT+8 -> Etc/GMT-8, +08:00 -> Etc/GMT-8 + if (substrait_func.arguments_size() != 2) + throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s must have 2 arguments", getName()); + + const substrait::Expression & arg1 = substrait_func.arguments()[1].value(); + if (!arg1.has_literal() || !arg1.literal().has_string()) + throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s 2nd argument should be string literal", getName()); + + const String & arg1_literal = arg1.literal().string(); + String time_zone_val = DateTimeUtil::convertTimeZone(arg1_literal); + auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); + auto nullable_string_type = DB::makeNullable(std::make_shared()); + const auto * time_zone_node = addColumnToActionsDAG(actions_dag, nullable_string_type, time_zone_val); + const auto * result_node = toFunctionNode(actions_dag, getCHFunctionName(substrait_func), {parsed_args[0], time_zone_node}); + return convertNodeTypeIfNeeded(substrait_func, result_node, actions_dag); + } +}; +} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 2c34baa6379a..a8a5a1e412a8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -300,9 +300,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") .excludeGlutenTest("unix_timestamp") .excludeGlutenTest("to_unix_timestamp") - .exclude("to_utc_timestamp with literal zone") .exclude("to_utc_timestamp with column zone") - .exclude("from_utc_timestamp with literal zone") .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest( diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index bb782fde3d4d..cb33f002553b 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -326,9 +326,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") .excludeGlutenTest("unix_timestamp") .excludeGlutenTest("to_unix_timestamp") - .exclude("to_utc_timestamp with literal zone") .exclude("to_utc_timestamp with column zone") - .exclude("from_utc_timestamp with literal zone") .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest( diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 7a38774273ac..07af1fa845ca 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -324,9 +324,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") .excludeGlutenTest("unix_timestamp") .excludeGlutenTest("to_unix_timestamp") - .exclude("to_utc_timestamp with literal zone") .exclude("to_utc_timestamp with column zone") - .exclude("from_utc_timestamp with literal zone") .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest( diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 7a38774273ac..07af1fa845ca 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -324,9 +324,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") .excludeGlutenTest("unix_timestamp") .excludeGlutenTest("to_unix_timestamp") - .exclude("to_utc_timestamp with literal zone") .exclude("to_utc_timestamp with column zone") - .exclude("from_utc_timestamp with literal zone") .exclude("from_utc_timestamp with column zone") enableSuite[GlutenDeprecatedAPISuite] enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff].excludeGlutenTest( From d3ccd4aea027455752d84f54bf1f3b589660bd4a Mon Sep 17 00:00:00 2001 From: Wenzheng Liu Date: Tue, 11 Jun 2024 17:57:01 +0800 Subject: [PATCH 242/402] [GLUTEN-5979][CH] Fix CHListenerApi initialize twice on spark local mode (#6037) --- .../clickhouse/CHListenerApi.scala | 6 +- .../GlutenClickHouseNativeLibSuite.scala | 79 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala index 665fdba88e55..43e0627dffef 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala @@ -48,7 +48,11 @@ class CHListenerApi extends ListenerApi with Logging { override def onExecutorStart(pc: PluginContext): Unit = { GlutenExecutorEndpoint.executorEndpoint = new GlutenExecutorEndpoint(pc.executorID, pc.conf) - initialize(pc.conf, isDriver = false) + if (pc.conf().get("spark.master").startsWith("local")) { + logDebug("Skipping duplicate initializing clickhouse backend on spark local mode") + } else { + initialize(pc.conf, isDriver = false) + } } override def onExecutorShutdown(): Unit = shutdown() diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala new file mode 100644 index 000000000000..0221f06bd681 --- /dev/null +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.gluten.GlutenConfig +import org.apache.gluten.exception.GlutenException +import org.apache.gluten.utils.UTSystemParameters + +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.PlanTest + +class GlutenClickHouseNativeLibSuite extends PlanTest { + + private def baseSparkConf: SparkConf = { + new SparkConf() + .set("spark.plugins", "org.apache.gluten.GlutenPlugin") + .set("spark.default.parallelism", "1") + .set("spark.memory.offHeap.enabled", "true") + .set("spark.memory.offHeap.size", "1024MB") + .set("spark.gluten.sql.enable.native.validation", "false") + } + + test("test columnar lib path not exist") { + var spark: SparkSession = null + try { + spark = SparkSession + .builder() + .master("local[1]") + .config(baseSparkConf) + .config(GlutenConfig.GLUTEN_LIB_PATH, "path/not/exist/libch.so") + .getOrCreate() + spark.sql("select 1").show() + } catch { + case e: Exception => + assert(e.isInstanceOf[GlutenException]) + assert( + e.getMessage.contains( + "library at path: path/not/exist/libch.so is not a file or does not exist")) + } finally { + if (spark != null) { + spark.stop() + } + } + } + + test("test CHListenerApi initialize only once") { + var spark: SparkSession = null + try { + spark = SparkSession + .builder() + .master("local[1]") + .config(baseSparkConf) + .config(GlutenConfig.GLUTEN_LIB_PATH, UTSystemParameters.clickHouseLibPath) + .config(GlutenConfig.GLUTEN_EXECUTOR_LIB_PATH, "/path/not/exist/libch.so") + .getOrCreate() + spark.sql("select 1").show() + } finally { + if (spark != null) { + spark.stop() + } + } + } + +} From 070022a9e5420cafb4688aab02abf6ad55ac0413 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Wed, 12 Jun 2024 09:19:15 +0800 Subject: [PATCH 243/402] [GLUTEN-6040][CH] Fix can't not load part after restart spark session (#6041) [CH] Fix can't not load part after restart spark session --- ...tenClickHouseMergeTreeWriteOnS3Suite.scala | 83 +++++++++++++++++-- .../Parser/MergeTreeRelParser.cpp | 2 +- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala index 44c2af76f933..c5dc3a23754e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala @@ -635,29 +635,102 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite } test("test mergetree insert with optimize basic") { - val table_name = "lineitem_mergetree_insert_optimize_basic_s3" - val dataPath = s"s3a://$BUCKET_NAME/$table_name" + val tableName = "lineitem_mergetree_insert_optimize_basic_s3" + val dataPath = s"s3a://$BUCKET_NAME/$tableName" withSQLConf( ("spark.databricks.delta.optimize.minFileSize" -> "200000000"), ("spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true") ) { spark.sql(s""" - |DROP TABLE IF EXISTS $table_name; + |DROP TABLE IF EXISTS $tableName; |""".stripMargin) spark.sql(s""" - |CREATE TABLE IF NOT EXISTS $table_name + |CREATE TABLE IF NOT EXISTS $tableName |USING clickhouse |LOCATION '$dataPath' | as select * from lineitem |""".stripMargin) - val ret = spark.sql(s"select count(*) from $table_name").collect() + val ret = spark.sql(s"select count(*) from $tableName").collect() assert(ret.apply(0).get(0) == 600572) assert( !new File(s"$CH_DEFAULT_STORAGE_DIR/lineitem_mergetree_insert_optimize_basic").exists()) } } + + test("test mergetree with primary keys pruning by driver") { + val tableName = "lineitem_mergetree_pk_pruning_by_driver_s3" + val dataPath = s"s3a://$BUCKET_NAME/$tableName" + spark.sql(s""" + |DROP TABLE IF EXISTS $tableName; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS $tableName + |( + | l_orderkey bigint, + | l_partkey bigint, + | l_suppkey bigint, + | l_linenumber bigint, + | l_quantity double, + | l_extendedprice double, + | l_discount double, + | l_tax double, + | l_returnflag string, + | l_linestatus string, + | l_shipdate date, + | l_commitdate date, + | l_receiptdate date, + | l_shipinstruct string, + | l_shipmode string, + | l_comment string + |) + |USING clickhouse + |TBLPROPERTIES (storage_policy='__s3_main', orderByKey='l_shipdate') + |LOCATION '$dataPath' + |""".stripMargin) + + spark.sql(s""" + | insert into table $tableName + | select * from lineitem + |""".stripMargin) + + FileUtils.forceDelete(new File(S3_METADATA_PATH)) + + val sqlStr = + s""" + |SELECT + | sum(l_extendedprice * l_discount) AS revenue + |FROM + | $tableName + |WHERE + | l_shipdate >= date'1994-01-01' + | AND l_shipdate < date'1994-01-01' + interval 1 year + | AND l_discount BETWEEN 0.06 - 0.01 AND 0.06 + 0.01 + | AND l_quantity < 24 + |""".stripMargin + + withSQLConf( + ("spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> "true")) { + runTPCHQueryBySQL(6, sqlStr) { + df => + val scanExec = collect(df.queryExecution.executedPlan) { + case f: FileSourceScanExecTransformer => f + } + assert(scanExec.size == 1) + + val mergetreeScan = scanExec(0) + assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) + + val plans = collect(df.queryExecution.executedPlan) { + case scanExec: BasicScanExecTransformer => scanExec + } + assert(plans.size == 1) + assert(plans(0).getSplitInfos.size == 1) + } + } + } } // scalastyle:off line.size.limit diff --git a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp index 9afa83973cd8..c36db6b7484a 100644 --- a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp +++ b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp @@ -398,7 +398,7 @@ String MergeTreeRelParser::filterRangesOnDriver(const substrait::ReadRel & read_ google::protobuf::StringValue table; table.ParseFromString(read_rel.advanced_extension().enhancement().value()); auto merge_tree_table = parseMergeTreeTableString(table.value()); - auto custom_storage_mergetree = parseStorage(merge_tree_table, global_context); + auto custom_storage_mergetree = parseStorage(merge_tree_table, global_context, true); auto input = TypeParser::buildBlockFromNamedStruct(read_rel.base_schema()); auto names_and_types_list = input.getNamesAndTypesList(); From 7e217cfdeae96337bcf59a082e66d5297f534f41 Mon Sep 17 00:00:00 2001 From: WangGuangxin Date: Wed, 12 Jun 2024 10:09:23 +0800 Subject: [PATCH 244/402] [GLUTEN-5625][VL] Support window range frame (#5626) --- .../clickhouse/CHSparkPlanExecApi.scala | 22 ++--- .../backendsapi/velox/VeloxBackend.scala | 20 ++--- .../gluten/execution/TestOperator.scala | 41 +++++++++- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 30 +++++-- cpp/velox/substrait/SubstraitToVeloxPlan.h | 6 ++ docs/developers/SubstraitModifications.md | 1 + .../expression/ExpressionBuilder.java | 22 +++-- .../expression/WindowFunctionNode.java | 62 +++++++++++--- .../substrait/proto/substrait/algebra.proto | 26 ++++-- .../backendsapi/BackendSettingsApi.scala | 2 + .../gluten/backendsapi/SparkPlanExecApi.scala | 35 ++++---- .../execution/WindowExecTransformer.scala | 13 --- .../columnar/rewrite/PullOutPreProject.scala | 15 +++- .../gluten/utils/PullOutProjectHelper.scala | 63 ++++++++++++++- .../PreComputeRangeFrameBound.scala | 80 +++++++++++++++++++ 15 files changed, 348 insertions(+), 90 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/PreComputeRangeFrameBound.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index a8a05c40f1cd..d7faa07a5a2e 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -66,6 +66,7 @@ import org.apache.commons.lang3.ClassUtils import java.lang.{Long => JLong} import java.util.{ArrayList => JArrayList, List => JList, Map => JMap} +import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer class CHSparkPlanExecApi extends SparkPlanExecApi { @@ -727,9 +728,10 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { new JArrayList[ExpressionNode](), columnName, ConverterUtils.getTypeNode(aggWindowFunc.dataType, aggWindowFunc.nullable), - WindowExecTransformer.getFrameBound(frame.upper), - WindowExecTransformer.getFrameBound(frame.lower), - frame.frameType.sql + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case aggExpression: AggregateExpression => @@ -753,9 +755,10 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { childrenNodeList, columnName, ConverterUtils.getTypeNode(aggExpression.dataType, aggExpression.nullable), - WindowExecTransformer.getFrameBound(frame.upper), - WindowExecTransformer.getFrameBound(frame.lower), - frame.frameType.sql + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case wf @ (Lead(_, _, _, _) | Lag(_, _, _, _)) => @@ -802,9 +805,10 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { childrenNodeList, columnName, ConverterUtils.getTypeNode(offsetWf.dataType, offsetWf.nullable), - WindowExecTransformer.getFrameBound(frame.upper), - WindowExecTransformer.getFrameBound(frame.lower), - frame.frameType.sql + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case _ => diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index f06929fff620..21e6246d1271 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -296,15 +296,9 @@ object VeloxBackendSettings extends BackendSettingsApi { case _ => throw new GlutenNotSupportException(s"$func is not supported.") } - // Block the offloading by checking Velox's current limitations - // when literal bound type is used for RangeFrame. def checkLimitations(swf: SpecifiedWindowFrame, orderSpec: Seq[SortOrder]): Unit = { - def doCheck(bound: Expression, isUpperBound: Boolean): Unit = { + def doCheck(bound: Expression): Unit = { bound match { - case e if e.foldable => - throw new GlutenNotSupportException( - "Window frame of type RANGE does" + - " not support constant arguments in velox backend") case _: SpecialFrameBoundary => case e if e.foldable => orderSpec.foreach( @@ -325,17 +319,11 @@ object VeloxBackendSettings extends BackendSettingsApi { "Only integral type & date type are" + " supported for sort key when literal bound type is used!") }) - val rawValue = e.eval().toString.toLong - if (isUpperBound && rawValue < 0) { - throw new GlutenNotSupportException("Negative upper bound is not supported!") - } else if (!isUpperBound && rawValue > 0) { - throw new GlutenNotSupportException("Positive lower bound is not supported!") - } case _ => } } - doCheck(swf.upper, true) - doCheck(swf.lower, false) + doCheck(swf.upper) + doCheck(swf.lower) } windowExpression.windowSpec.frameSpecification match { @@ -495,4 +483,6 @@ object VeloxBackendSettings extends BackendSettingsApi { override def supportColumnarArrowUdf(): Boolean = true override def generateHdfsConfForLibhdfs(): Boolean = true + + override def needPreComputeRangeFrameBoundary(): Boolean = true } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index ae8d64a09937..3cf485aac06b 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -212,17 +212,56 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla Seq("sort", "streaming").foreach { windowType => withSQLConf("spark.gluten.sql.columnar.backend.velox.window.type" -> windowType) { + runQueryAndCompare( + "select max(l_partkey) over" + + " (partition by l_suppkey order by l_orderkey" + + " RANGE BETWEEN 1 PRECEDING AND CURRENT ROW), " + + "min(l_comment) over" + + " (partition by l_suppkey order by l_linenumber" + + " RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) from lineitem ") { + checkSparkOperatorMatch[WindowExecTransformer] + } + runQueryAndCompare( "select max(l_partkey) over" + " (partition by l_suppkey order by l_orderkey" + " RANGE BETWEEN CURRENT ROW AND 2 FOLLOWING) from lineitem ") { - checkSparkOperatorMatch[WindowExec] + checkSparkOperatorMatch[WindowExecTransformer] } runQueryAndCompare( "select max(l_partkey) over" + " (partition by l_suppkey order by l_orderkey" + " RANGE BETWEEN 6 PRECEDING AND CURRENT ROW) from lineitem ") { + checkSparkOperatorMatch[WindowExecTransformer] + } + + runQueryAndCompare( + "select max(l_partkey) over" + + " (partition by l_suppkey order by l_orderkey" + + " RANGE BETWEEN 6 PRECEDING AND 2 FOLLOWING) from lineitem ") { + checkSparkOperatorMatch[WindowExecTransformer] + } + + runQueryAndCompare( + "select max(l_partkey) over" + + " (partition by l_suppkey order by l_orderkey" + + " RANGE BETWEEN 6 PRECEDING AND 3 PRECEDING) from lineitem ") { + checkSparkOperatorMatch[WindowExecTransformer] + } + + runQueryAndCompare( + "select max(l_partkey) over" + + " (partition by l_suppkey order by l_orderkey" + + " RANGE BETWEEN 3 FOLLOWING AND 6 FOLLOWING) from lineitem ") { + checkSparkOperatorMatch[WindowExecTransformer] + } + + // DecimalType as order by column is not supported + runQueryAndCompare( + "select min(l_comment) over" + + " (partition by l_suppkey order by l_discount" + + " RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) from lineitem ") { checkSparkOperatorMatch[WindowExec] } diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index b82eead2c565..4e875d4790e5 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -823,10 +823,11 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: nextPlanNodeId(), replicated, unnest, std::move(unnestNames), ordinalityName, childNode); } -const core::WindowNode::Frame createWindowFrame( +const core::WindowNode::Frame SubstraitToVeloxPlanConverter::createWindowFrame( const ::substrait::Expression_WindowFunction_Bound& lower_bound, const ::substrait::Expression_WindowFunction_Bound& upper_bound, - const ::substrait::WindowType& type) { + const ::substrait::WindowType& type, + const RowTypePtr& inputType) { core::WindowNode::Frame frame; switch (type) { case ::substrait::WindowType::ROWS: @@ -839,9 +840,22 @@ const core::WindowNode::Frame createWindowFrame( VELOX_FAIL("the window type only support ROWS and RANGE, and the input type is ", std::to_string(type)); } - auto boundTypeConversion = [](::substrait::Expression_WindowFunction_Bound boundType) + auto specifiedBound = + [&](bool hasOffset, int64_t offset, const ::substrait::Expression& columnRef) -> core::TypedExprPtr { + if (hasOffset) { + VELOX_CHECK( + frame.type != core::WindowNode::WindowType::kRange, + "for RANGE frame offset, we should pre-calculate the range frame boundary and pass the column reference, but got a constant offset.") + return std::make_shared(BIGINT(), variant(offset)); + } else { + VELOX_CHECK( + frame.type != core::WindowNode::WindowType::kRows, "for ROW frame offset, we should pass a constant offset.") + return exprConverter_->toVeloxExpr(columnRef, inputType); + } + }; + + auto boundTypeConversion = [&](::substrait::Expression_WindowFunction_Bound boundType) -> std::tuple { - // TODO: support non-literal expression. if (boundType.has_current_row()) { return std::make_tuple(core::WindowNode::BoundType::kCurrentRow, nullptr); } else if (boundType.has_unbounded_following()) { @@ -849,13 +863,15 @@ const core::WindowNode::Frame createWindowFrame( } else if (boundType.has_unbounded_preceding()) { return std::make_tuple(core::WindowNode::BoundType::kUnboundedPreceding, nullptr); } else if (boundType.has_following()) { + auto following = boundType.following(); return std::make_tuple( core::WindowNode::BoundType::kFollowing, - std::make_shared(BIGINT(), variant(boundType.following().offset()))); + specifiedBound(following.has_offset(), following.offset(), following.ref())); } else if (boundType.has_preceding()) { + auto preceding = boundType.preceding(); return std::make_tuple( core::WindowNode::BoundType::kPreceding, - std::make_shared(BIGINT(), variant(boundType.preceding().offset()))); + specifiedBound(preceding.has_offset(), preceding.offset(), preceding.ref())); } else { VELOX_FAIL("The BoundType is not supported."); } @@ -906,7 +922,7 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: windowColumnNames.push_back(windowFunction.column_name()); windowNodeFunctions.push_back( - {std::move(windowCall), std::move(createWindowFrame(lowerBound, upperBound, type)), ignoreNulls}); + {std::move(windowCall), std::move(createWindowFrame(lowerBound, upperBound, type, inputType)), ignoreNulls}); } // Construct partitionKeys diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.h b/cpp/velox/substrait/SubstraitToVeloxPlan.h index 567ebb215078..3a0e677afeaa 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.h +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.h @@ -555,6 +555,12 @@ class SubstraitToVeloxPlanConverter { return toVeloxPlan(rel.input()); } + const core::WindowNode::Frame createWindowFrame( + const ::substrait::Expression_WindowFunction_Bound& lower_bound, + const ::substrait::Expression_WindowFunction_Bound& upper_bound, + const ::substrait::WindowType& type, + const RowTypePtr& inputType); + /// The unique identification for each PlanNode. int planNodeId_ = 0; diff --git a/docs/developers/SubstraitModifications.md b/docs/developers/SubstraitModifications.md index 38406425af96..24a9c1a2128d 100644 --- a/docs/developers/SubstraitModifications.md +++ b/docs/developers/SubstraitModifications.md @@ -27,6 +27,7 @@ changed `Unbounded` in `WindowFunction` into `Unbounded_Preceding` and `Unbounde * Added `PartitionColumn` in `LocalFiles`([#2405](https://github.com/apache/incubator-gluten/pull/2405)). * Added `WriteRel` ([#3690](https://github.com/apache/incubator-gluten/pull/3690)). * Added `TopNRel` ([#5409](https://github.com/apache/incubator-gluten/pull/5409)). +* Added `ref` field in window bound `Preceding` and `Following` ([#5626](https://github.com/apache/incubator-gluten/pull/5626)). ## Modifications to type.proto diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java b/gluten-core/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java index 5d106938cef5..e322e1528cac 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java @@ -21,6 +21,8 @@ import org.apache.gluten.substrait.type.*; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.expressions.Attribute; +import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.util.ArrayData; import org.apache.spark.sql.catalyst.util.MapData; import org.apache.spark.sql.types.*; @@ -264,9 +266,10 @@ public static WindowFunctionNode makeWindowFunction( List expressionNodes, String columnName, TypeNode outputTypeNode, - String upperBound, - String lowerBound, - String frameType) { + Expression upperBound, + Expression lowerBound, + String frameType, + List originalInputAttributes) { return makeWindowFunction( functionId, expressionNodes, @@ -275,7 +278,8 @@ public static WindowFunctionNode makeWindowFunction( upperBound, lowerBound, frameType, - false); + false, + originalInputAttributes); } public static WindowFunctionNode makeWindowFunction( @@ -283,10 +287,11 @@ public static WindowFunctionNode makeWindowFunction( List expressionNodes, String columnName, TypeNode outputTypeNode, - String upperBound, - String lowerBound, + Expression upperBound, + Expression lowerBound, String frameType, - boolean ignoreNulls) { + boolean ignoreNulls, + List originalInputAttributes) { return new WindowFunctionNode( functionId, expressionNodes, @@ -295,6 +300,7 @@ public static WindowFunctionNode makeWindowFunction( upperBound, lowerBound, frameType, - ignoreNulls); + ignoreNulls, + originalInputAttributes); } } diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/expression/WindowFunctionNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/expression/WindowFunctionNode.java index 67d0d6e575ff..b9f1fbc126cc 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/expression/WindowFunctionNode.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/expression/WindowFunctionNode.java @@ -16,17 +16,24 @@ */ package org.apache.gluten.substrait.expression; +import org.apache.gluten.exception.GlutenException; +import org.apache.gluten.expression.ExpressionConverter; import org.apache.gluten.substrait.type.TypeNode; import io.substrait.proto.Expression; import io.substrait.proto.FunctionArgument; import io.substrait.proto.FunctionOption; import io.substrait.proto.WindowType; +import org.apache.spark.sql.catalyst.expressions.Attribute; +import org.apache.spark.sql.catalyst.expressions.PreComputeRangeFrameBound; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import scala.collection.JavaConverters; + public class WindowFunctionNode implements Serializable { private final Integer functionId; private final List expressionNodes = new ArrayList<>(); @@ -34,23 +41,26 @@ public class WindowFunctionNode implements Serializable { private final String columnName; private final TypeNode outputTypeNode; - private final String upperBound; + private final org.apache.spark.sql.catalyst.expressions.Expression upperBound; - private final String lowerBound; + private final org.apache.spark.sql.catalyst.expressions.Expression lowerBound; private final String frameType; private final boolean ignoreNulls; + private final List originalInputAttributes; + WindowFunctionNode( Integer functionId, List expressionNodes, String columnName, TypeNode outputTypeNode, - String upperBound, - String lowerBound, + org.apache.spark.sql.catalyst.expressions.Expression upperBound, + org.apache.spark.sql.catalyst.expressions.Expression lowerBound, String frameType, - boolean ignoreNulls) { + boolean ignoreNulls, + List originalInputAttributes) { this.functionId = functionId; this.expressionNodes.addAll(expressionNodes); this.columnName = columnName; @@ -59,11 +69,13 @@ public class WindowFunctionNode implements Serializable { this.lowerBound = lowerBound; this.frameType = frameType; this.ignoreNulls = ignoreNulls; + this.originalInputAttributes = originalInputAttributes; } private Expression.WindowFunction.Bound.Builder setBound( - Expression.WindowFunction.Bound.Builder builder, String boundType) { - switch (boundType) { + Expression.WindowFunction.Bound.Builder builder, + org.apache.spark.sql.catalyst.expressions.Expression boundType) { + switch (boundType.sql()) { case ("CURRENT ROW"): Expression.WindowFunction.Bound.CurrentRow.Builder currentRowBuilder = Expression.WindowFunction.Bound.CurrentRow.newBuilder(); @@ -80,8 +92,36 @@ private Expression.WindowFunction.Bound.Builder setBound( builder.setUnboundedFollowing(followingBuilder.build()); break; default: - try { - Long offset = Long.valueOf(boundType); + if (boundType instanceof PreComputeRangeFrameBound) { + // Used only when backend is velox and frame type is RANGE. + if (!frameType.equals("RANGE")) { + throw new GlutenException( + "Only Range frame supports PreComputeRangeFrameBound, but got " + frameType); + } + ExpressionNode refNode = + ExpressionConverter.replaceWithExpressionTransformer( + ((PreComputeRangeFrameBound) boundType).child().toAttribute(), + JavaConverters.asScalaIteratorConverter(originalInputAttributes.iterator()) + .asScala() + .toSeq()) + .doTransform(new HashMap()); + Long offset = Long.valueOf(boundType.eval(null).toString()); + if (offset < 0) { + Expression.WindowFunction.Bound.Preceding.Builder refPrecedingBuilder = + Expression.WindowFunction.Bound.Preceding.newBuilder(); + refPrecedingBuilder.setRef(refNode.toProtobuf()); + builder.setPreceding(refPrecedingBuilder.build()); + } else { + Expression.WindowFunction.Bound.Following.Builder refFollowingBuilder = + Expression.WindowFunction.Bound.Following.newBuilder(); + refFollowingBuilder.setRef(refNode.toProtobuf()); + builder.setFollowing(refFollowingBuilder.build()); + } + } else if (boundType.foldable()) { + // Used when + // 1. Velox backend and frame type is ROW + // 2. Clickhouse backend + Long offset = Long.valueOf(boundType.eval(null).toString()); if (offset < 0) { Expression.WindowFunction.Bound.Preceding.Builder offsetPrecedingBuilder = Expression.WindowFunction.Bound.Preceding.newBuilder(); @@ -93,9 +133,9 @@ private Expression.WindowFunction.Bound.Builder setBound( offsetFollowingBuilder.setOffset(offset); builder.setFollowing(offsetFollowingBuilder.build()); } - } catch (NumberFormatException e) { + } else { throw new UnsupportedOperationException( - "Unsupported Window Function Frame Type:" + boundType); + "Unsupported Window Function Frame Bound Type: " + boundType); } } return builder; diff --git a/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto b/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto index 877493439f95..0e51baf5ad4c 100644 --- a/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto +++ b/gluten-core/src/main/resources/substrait/proto/substrait/algebra.proto @@ -996,18 +996,28 @@ message Expression { message Bound { // Defines that the bound extends this far back from the current record. message Preceding { - // A strictly positive integer specifying the number of records that - // the window extends back from the current record. Required. Use - // CurrentRow for offset zero and Following for negative offsets. - int64 offset = 1; + oneof kind { + // A strictly positive integer specifying the number of records that + // the window extends back from the current record. Use + // CurrentRow for offset zero and Following for negative offsets. + int64 offset = 1; + + // the reference to pre-project range frame boundary. + Expression ref = 2; + } } // Defines that the bound extends this far ahead of the current record. message Following { - // A strictly positive integer specifying the number of records that - // the window extends ahead of the current record. Required. Use - // CurrentRow for offset zero and Preceding for negative offsets. - int64 offset = 1; + oneof kind { + // A strictly positive integer specifying the number of records that + // the window extends ahead of the current record. Use + // CurrentRow for offset zero and Preceding for negative offsets. + int64 offset = 1; + + // the reference to pre-project range frame boundary. + Expression ref = 2; + } } // Defines that the bound extends to or from the current record. diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index d18273af2faa..b7a3bc1b6ef2 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -152,4 +152,6 @@ trait BackendSettingsApi { def supportColumnarArrowUdf(): Boolean = false def generateHdfsConfForLibhdfs(): Boolean = false + + def needPreComputeRangeFrameBoundary(): Boolean = false } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 8a1baae51092..8bc8e136bd5d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -529,9 +529,10 @@ trait SparkPlanExecApi { new JArrayList[ExpressionNode](), columnName, ConverterUtils.getTypeNode(aggWindowFunc.dataType, aggWindowFunc.nullable), - WindowExecTransformer.getFrameBound(frame.upper), - WindowExecTransformer.getFrameBound(frame.lower), - frame.frameType.sql + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case aggExpression: AggregateExpression => @@ -554,9 +555,10 @@ trait SparkPlanExecApi { childrenNodeList, columnName, ConverterUtils.getTypeNode(aggExpression.dataType, aggExpression.nullable), - WindowExecTransformer.getFrameBound(frame.upper), - WindowExecTransformer.getFrameBound(frame.lower), - frame.frameType.sql + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case wf @ (_: Lead | _: Lag) => @@ -590,10 +592,11 @@ trait SparkPlanExecApi { childrenNodeList, columnName, ConverterUtils.getTypeNode(offsetWf.dataType, offsetWf.nullable), - WindowExecTransformer.getFrameBound(frame.upper), - WindowExecTransformer.getFrameBound(frame.lower), + frame.upper, + frame.lower, frame.frameType.sql, - offsetWf.ignoreNulls + offsetWf.ignoreNulls, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case wf @ NthValue(input, offset: Literal, ignoreNulls: Boolean) => @@ -609,10 +612,11 @@ trait SparkPlanExecApi { childrenNodeList, columnName, ConverterUtils.getTypeNode(wf.dataType, wf.nullable), - frame.upper.sql, - frame.lower.sql, + frame.upper, + frame.lower, frame.frameType.sql, - ignoreNulls + ignoreNulls, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case wf @ NTile(buckets: Expression) => @@ -625,9 +629,10 @@ trait SparkPlanExecApi { childrenNodeList, columnName, ConverterUtils.getTypeNode(wf.dataType, wf.nullable), - frame.upper.sql, - frame.lower.sql, - frame.frameType.sql + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) case _ => diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala index ef6a767b5604..6832221a404d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala @@ -197,16 +197,3 @@ case class WindowExecTransformer( override protected def withNewChildInternal(newChild: SparkPlan): WindowExecTransformer = copy(child = newChild) } - -object WindowExecTransformer { - - /** Gets lower/upper bound represented in string. */ - def getFrameBound(bound: Expression): String = { - // The lower/upper can be either a foldable Expression or a SpecialFrameBoundary. - if (bound.foldable) { - bound.eval().toString - } else { - bound.sql - } - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala index 50dc55423605..73b8ab2607eb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/PullOutPreProject.scala @@ -75,6 +75,17 @@ object PullOutPreProject extends RewriteSingleNode with PullOutProjectHelper { case _ => false } case _ => false + }.isDefined) || + window.windowExpression.exists(_.find { + case we: WindowExpression => + we.windowSpec.frameSpecification match { + case swf: SpecifiedWindowFrame + if needPreComputeRangeFrame(swf) && supportPreComputeRangeFrame( + we.windowSpec.orderSpec) => + true + case _ => false + } + case _ => false }.isDefined) case plan if SparkShimLoader.getSparkShims.isWindowGroupLimitExec(plan) => val window = SparkShimLoader.getSparkShims @@ -174,7 +185,9 @@ object PullOutPreProject extends RewriteSingleNode with PullOutProjectHelper { // Handle windowExpressions. val newWindowExpressions = window.windowExpression.toIndexedSeq.map { - _.transform { case we: WindowExpression => rewriteWindowExpression(we, expressionMap) } + _.transform { + case we: WindowExpression => rewriteWindowExpression(we, newOrderSpec, expressionMap) + } } val newWindow = window.copy( diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala index 505f13f263a2..12055f9e9721 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala @@ -16,11 +16,13 @@ */ package org.apache.gluten.utils -import org.apache.gluten.exception.GlutenNotSupportException +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.exception.{GlutenException, GlutenNotSupportException} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction} import org.apache.spark.sql.execution.aggregate._ +import org.apache.spark.sql.types.{ByteType, DateType, IntegerType, LongType, ShortType} import java.util.concurrent.atomic.AtomicInteger @@ -143,8 +145,49 @@ trait PullOutProjectHelper { ae.copy(aggregateFunction = newAggFunc, filter = newFilter) } + private def needPreComputeRangeFrameBoundary(bound: Expression): Boolean = { + bound match { + case _: PreComputeRangeFrameBound => false + case _ if !bound.foldable => false + case _ => true + } + } + + private def preComputeRangeFrameBoundary( + bound: Expression, + orderSpec: SortOrder, + expressionMap: mutable.HashMap[Expression, NamedExpression]): Expression = { + bound match { + case _: PreComputeRangeFrameBound => bound + case _ if !bound.foldable => bound + case _ if bound.foldable => + val a = expressionMap + .getOrElseUpdate( + bound.canonicalized, + Alias(Add(orderSpec.child, bound), generatePreAliasName)()) + PreComputeRangeFrameBound(a.asInstanceOf[Alias], bound) + } + } + + protected def needPreComputeRangeFrame(swf: SpecifiedWindowFrame): Boolean = { + BackendsApiManager.getSettings.needPreComputeRangeFrameBoundary && + swf.frameType == RangeFrame && + (needPreComputeRangeFrameBoundary(swf.lower) || needPreComputeRangeFrameBoundary(swf.upper)) + } + + protected def supportPreComputeRangeFrame(sortOrders: Seq[SortOrder]): Boolean = { + sortOrders.forall { + _.dataType match { + case ByteType | ShortType | IntegerType | LongType | DateType => true + // Only integral type & date type are supported for sort key with Range Frame + case _ => false + } + } + } + protected def rewriteWindowExpression( we: WindowExpression, + orderSpecs: Seq[SortOrder], expressionMap: mutable.HashMap[Expression, NamedExpression]): WindowExpression = { val newWindowFunc = we.windowFunction match { case windowFunc: WindowFunction => @@ -156,6 +199,22 @@ trait PullOutProjectHelper { case ae: AggregateExpression => rewriteAggregateExpression(ae, expressionMap) case other => other } - we.copy(windowFunction = newWindowFunc) + + val newWindowSpec = we.windowSpec.frameSpecification match { + case swf: SpecifiedWindowFrame if needPreComputeRangeFrame(swf) => + // This is guaranteed by Spark, but we still check it here + if (orderSpecs.size != 1) { + throw new GlutenException( + s"A range window frame with value boundaries expects one and only one " + + s"order by expression: ${orderSpecs.mkString(",")}") + } + val orderSpec = orderSpecs.head + val lowerFrameCol = preComputeRangeFrameBoundary(swf.lower, orderSpec, expressionMap) + val upperFrameCol = preComputeRangeFrameBoundary(swf.upper, orderSpec, expressionMap) + val newFrame = swf.copy(lower = lowerFrameCol, upper = upperFrameCol) + we.windowSpec.copy(frameSpecification = newFrame) + case _ => we.windowSpec + } + we.copy(windowFunction = newWindowFunc, windowSpec = newWindowSpec) } } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/PreComputeRangeFrameBound.scala b/gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/PreComputeRangeFrameBound.scala new file mode 100644 index 000000000000..73c1cb3de609 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/spark/sql/catalyst/expressions/PreComputeRangeFrameBound.scala @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.types.{DataType, Metadata} + +/** + * Represents a pre-compute boundary for range frame when boundary is non-SpecialFrameBoundary, + * since Velox doesn't support constant offset for range frame. It acts like the original boundary + * which is foldable and generate the same result when eval is invoked so that if the WindowExec + * fallback to Vanilla Spark it can still work correctly. + * @param child + * The alias to pre-compute projection column + * @param originalBound + * The original boundary which is a foldable expression + */ +case class PreComputeRangeFrameBound(child: Alias, originalBound: Expression) + extends UnaryExpression + with NamedExpression { + + override def foldable: Boolean = true + + override def eval(input: InternalRow): Any = originalBound.eval(input) + + override def genCode(ctx: CodegenContext): ExprCode = originalBound.genCode(ctx) + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = + originalBound.genCode(ctx) + + override def name: String = child.name + + override def exprId: ExprId = child.exprId + + override def qualifier: Seq[String] = child.qualifier + + override def newInstance(): NamedExpression = + PreComputeRangeFrameBound(child.newInstance().asInstanceOf[Alias], originalBound) + + override lazy val resolved: Boolean = originalBound.resolved + + override def dataType: DataType = child.dataType + + override def nullable: Boolean = child.nullable + + override def metadata: Metadata = child.metadata + + override def toAttribute: Attribute = child.toAttribute + + override def toString: String = child.toString + + override def hashCode(): Int = child.hashCode() + + override def equals(other: Any): Boolean = other match { + case a: PreComputeRangeFrameBound => + child.equals(a.child) + case _ => false + } + + override def sql: String = child.sql + + override protected def withNewChildInternal(newChild: Expression): PreComputeRangeFrameBound = + copy(child = newChild.asInstanceOf[Alias]) + +} From 07c09b5868421b4195f3e666aa7950f0d2312213 Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Wed, 12 Jun 2024 10:17:36 +0800 Subject: [PATCH 245/402] [CH] add throttler to GlutenHDFSDisk (#6046) [CH] add throttler to GlutenHDFSDisk Co-authored-by: liuneng1994 --- .../Disks/ObjectStorages/GlutenDiskHDFS.cpp | 13 ++++++++++++- .../Disks/ObjectStorages/GlutenDiskHDFS.h | 8 ++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp index bff4108f28a1..cdbe6c72897c 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp @@ -17,6 +17,8 @@ #include "GlutenDiskHDFS.h" #include + +#include #include #if USE_HDFS @@ -70,6 +72,15 @@ DiskObjectStoragePtr GlutenDiskHDFS::createDiskObjectStorage() config_prefix); } - +std::unique_ptr GlutenDiskHDFS::writeFile( + const String & path, + size_t buf_size, + DB::WriteMode mode, + const DB::WriteSettings & settings) +{ + if (throttler) + throttler->add(1); + return DiskObjectStorage::writeFile(path, buf_size, mode, settings); +} } #endif \ No newline at end of file diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h index 9caedaae8785..4e375b283951 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h @@ -19,6 +19,7 @@ #include +#include #include #if USE_HDFS #include @@ -43,6 +44,8 @@ class GlutenDiskHDFS : public DB::DiskObjectStorage object_key_prefix = object_key_prefix_; hdfs_object_storage = dynamic_cast(object_storage_.get()); hdfsSetWorkingDirectory(hdfs_object_storage->getHDFSFS(), "/"); + auto max_speed = config.getUInt(config_prefix + ".write_speed", 450); + throttler = std::make_shared(max_speed); } void createDirectory(const String & path) override; @@ -52,11 +55,16 @@ class GlutenDiskHDFS : public DB::DiskObjectStorage void removeDirectory(const String & path) override; DB::DiskObjectStoragePtr createDiskObjectStorage() override; + + std::unique_ptr writeFile(const String& path, size_t buf_size, DB::WriteMode mode, + const DB::WriteSettings& settings) override; + private: String path2AbsPath(const String & path); GlutenHDFSObjectStorage * hdfs_object_storage; String object_key_prefix; + DB::ThrottlerPtr throttler; }; #endif } From f610059f1ded8c0942ee8dd0b82b0223d3180a40 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 12 Jun 2024 10:22:26 +0800 Subject: [PATCH 246/402] [CORE] Rework Gluten + DPP compatibility (#6035) --- .../DataSourceScanTransformerRegister.scala | 8 +- .../execution/ScanTransformerFactory.scala | 42 ++------ .../expression/ExpressionConverter.scala | 101 +---------------- .../columnar/MiscColumnarRules.scala | 102 +++++++++++++++++- .../columnar/OffloadSingleNode.scala | 68 ++---------- .../columnar/TransformHintRule.scala | 2 +- .../enumerated/EnumeratedApplier.scala | 5 +- .../columnar/heuristic/HeuristicApplier.scala | 5 +- .../ColumnarSubqueryBroadcastExec.scala | 50 ++++----- .../execution/DeltaScanTransformer.scala | 6 +- .../DeltaScanTransformerProvider.scala | 6 +- .../execution/IcebergScanTransformer.scala | 6 +- .../IcebergTransformerProvider.scala | 6 +- 13 files changed, 153 insertions(+), 254 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/DataSourceScanTransformerRegister.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/DataSourceScanTransformerRegister.scala index b899790c3472..5b46c23857d3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/DataSourceScanTransformerRegister.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/DataSourceScanTransformerRegister.scala @@ -16,7 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.execution.datasources.v2.BatchScanExec @@ -44,15 +43,12 @@ trait DataSourceScanTransformerRegister { val scanClassName: String def createDataSourceTransformer( - batchScan: FileSourceScanExec, - newPartitionFilters: Seq[Expression]): FileSourceScanExecTransformerBase = { + batchScan: FileSourceScanExec): FileSourceScanExecTransformerBase = { throw new UnsupportedOperationException( "This should not be called, please implement this method in child class."); } - def createDataSourceV2Transformer( - batchScan: BatchScanExec, - newPartitionFilters: Seq[Expression]): BatchScanExecTransformerBase = { + def createDataSourceV2Transformer(batchScan: BatchScanExec): BatchScanExecTransformerBase = { throw new UnsupportedOperationException( "This should not be called, please implement this method in child class."); } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala index fc2b8f506492..fcb9e983e76b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala @@ -17,7 +17,6 @@ package org.apache.gluten.execution import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.expression.ExpressionConverter import org.apache.gluten.extension.columnar.TransformHints import org.apache.gluten.sql.shims.SparkShimLoader @@ -37,14 +36,7 @@ object ScanTransformerFactory { def createFileSourceScanTransformer( scanExec: FileSourceScanExec, - allPushDownFilters: Option[Seq[Expression]] = None, - validation: Boolean = false): FileSourceScanExecTransformerBase = { - // transform BroadcastExchangeExec to ColumnarBroadcastExchangeExec in partitionFilters - val newPartitionFilters = if (validation) { - scanExec.partitionFilters - } else { - ExpressionConverter.transformDynamicPruningExpr(scanExec.partitionFilters) - } + allPushDownFilters: Option[Seq[Expression]] = None): FileSourceScanExecTransformerBase = { val fileFormat = scanExec.relation.fileFormat lookupDataSourceScanTransformer(fileFormat.getClass.getName) match { case Some(clz) => @@ -52,13 +44,13 @@ object ScanTransformerFactory { .getDeclaredConstructor() .newInstance() .asInstanceOf[DataSourceScanTransformerRegister] - .createDataSourceTransformer(scanExec, newPartitionFilters) + .createDataSourceTransformer(scanExec) case _ => - new FileSourceScanExecTransformer( + FileSourceScanExecTransformer( scanExec.relation, scanExec.output, scanExec.requiredSchema, - newPartitionFilters, + scanExec.partitionFilters, scanExec.optionalBucketSet, scanExec.optionalNumCoalescedBuckets, allPushDownFilters.getOrElse(scanExec.dataFilters), @@ -69,8 +61,7 @@ object ScanTransformerFactory { } private def lookupBatchScanTransformer( - batchScanExec: BatchScanExec, - newPartitionFilters: Seq[Expression]): BatchScanExecTransformerBase = { + batchScanExec: BatchScanExec): BatchScanExecTransformerBase = { val scan = batchScanExec.scan lookupDataSourceScanTransformer(scan.getClass.getName) match { case Some(clz) => @@ -78,14 +69,14 @@ object ScanTransformerFactory { .getDeclaredConstructor() .newInstance() .asInstanceOf[DataSourceScanTransformerRegister] - .createDataSourceV2Transformer(batchScanExec, newPartitionFilters) + .createDataSourceV2Transformer(batchScanExec) case _ => scan match { case _: FileScan => - new BatchScanExecTransformer( + BatchScanExecTransformer( batchScanExec.output, batchScanExec.scan, - newPartitionFilters, + batchScanExec.runtimeFilters, table = SparkShimLoader.getSparkShims.getBatchScanExecTable(batchScanExec) ) case _ => @@ -99,14 +90,7 @@ object ScanTransformerFactory { allPushDownFilters: Option[Seq[Expression]] = None, validation: Boolean = false): SparkPlan = { if (supportedBatchScan(batchScan.scan)) { - val newPartitionFilters = if (validation) { - // No transformation is needed for DynamicPruningExpressions - // during the validation process. - batchScan.runtimeFilters - } else { - ExpressionConverter.transformDynamicPruningExpr(batchScan.runtimeFilters) - } - val transformer = lookupBatchScanTransformer(batchScan, newPartitionFilters) + val transformer = lookupBatchScanTransformer(batchScan) if (!validation && allPushDownFilters.isDefined) { transformer.setPushDownFilters(allPushDownFilters.get) // Validate again if allPushDownFilters is defined. @@ -125,12 +109,8 @@ object ScanTransformerFactory { if (validation) { throw new GlutenNotSupportException(s"Unsupported scan ${batchScan.scan}") } - // If filter expressions aren't empty, we need to transform the inner operators, - // and fallback the BatchScanExec itself. - val newSource = batchScan.copy(runtimeFilters = ExpressionConverter - .transformDynamicPruningExpr(batchScan.runtimeFilters)) - TransformHints.tagNotTransformable(newSource, "The scan in BatchScanExec is not supported.") - newSource + TransformHints.tagNotTransformable(batchScan, "The scan in BatchScanExec is not supported.") + batchScan } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 9ebe44f6ca54..5d0af9e526ee 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -16,10 +16,8 @@ */ package org.apache.gluten.expression -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.test.TestStats import org.apache.gluten.utils.DecimalArithmeticUtil @@ -29,9 +27,7 @@ import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero -import org.apache.spark.sql.execution.{ScalarSubquery, _} -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec -import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec +import org.apache.spark.sql.execution.ScalarSubquery import org.apache.spark.sql.hive.HiveUDFTransformer import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -680,99 +676,4 @@ object ExpressionConverter extends SQLConfHelper with Logging { } substraitExprName } - - /** - * Transform BroadcastExchangeExec to ColumnarBroadcastExchangeExec in DynamicPruningExpression. - * - * @param partitionFilters - * The partition filter of Scan - * @return - * Transformed partition filter - */ - def transformDynamicPruningExpr(partitionFilters: Seq[Expression]): Seq[Expression] = { - - def convertBroadcastExchangeToColumnar( - exchange: BroadcastExchangeExec): ColumnarBroadcastExchangeExec = { - val newChild = Transitions.toBackendBatchPlan(exchange.child) - ColumnarBroadcastExchangeExec(exchange.mode, newChild) - } - - if ( - GlutenConfig.getConf.enableScanOnly || !GlutenConfig.getConf.enableColumnarBroadcastExchange - ) { - // Disable ColumnarSubqueryBroadcast for scan-only execution - // or ColumnarBroadcastExchange was disabled. - partitionFilters - } else { - partitionFilters.map { - case dynamicPruning: DynamicPruningExpression => - dynamicPruning.transform { - // Lookup inside subqueries for duplicate exchanges. - case in: InSubqueryExec => - in.plan match { - case s: SubqueryBroadcastExec => - val newIn = s - .transform { - case exchange: BroadcastExchangeExec => - convertBroadcastExchangeToColumnar(exchange) - } - .asInstanceOf[SubqueryBroadcastExec] - val transformSubqueryBroadcast = ColumnarSubqueryBroadcastExec( - newIn.name, - newIn.index, - newIn.buildKeys, - newIn.child) - - // When AQE is on, spark will apply ReuseAdaptiveSubquery rule first, - // it will reuse vanilla SubqueryBroadcastExec, - // and then use gluten ColumnarOverrides rule to transform Subquery, - // so all the SubqueryBroadcastExec in the ReusedSubqueryExec will be transformed - // to a new ColumnarSubqueryBroadcastExec for each SubqueryBroadcastExec, - // which will lead to execute ColumnarSubqueryBroadcastExec.relationFuture - // repeatedly even in the ReusedSubqueryExec. - // - // On the other hand, it needs to use - // the AdaptiveSparkPlanExec.AdaptiveExecutionContext to hold the reused map - // for each query. - newIn.child match { - case a: AdaptiveSparkPlanExec if SQLConf.get.subqueryReuseEnabled => - // When AQE is on and reuseSubquery is on. - a.context.subqueryCache - .update(newIn.canonicalized, transformSubqueryBroadcast) - case _ => - } - in.copy(plan = transformSubqueryBroadcast.asInstanceOf[BaseSubqueryExec]) - case r: ReusedSubqueryExec if r.child.isInstanceOf[SubqueryBroadcastExec] => - val newIn = r.child - .transform { - case exchange: BroadcastExchangeExec => - convertBroadcastExchangeToColumnar(exchange) - } - .asInstanceOf[SubqueryBroadcastExec] - newIn.child match { - case a: AdaptiveSparkPlanExec => - // Only when AQE is on, it needs to replace SubqueryBroadcastExec - // with reused ColumnarSubqueryBroadcastExec - val cachedSubquery = a.context.subqueryCache.get(newIn.canonicalized) - if (cachedSubquery.isDefined) { - in.copy(plan = ReusedSubqueryExec(cachedSubquery.get)) - } else { - val errMsg = "Can not get the reused ColumnarSubqueryBroadcastExec" + - "by the ${newIn.canonicalized}" - logWarning(errMsg) - throw new UnsupportedOperationException(errMsg) - } - case _ => - val errMsg = "Can not get the reused ColumnarSubqueryBroadcastExec" + - "by the ${newIn.canonicalized}" - logWarning(errMsg) - throw new UnsupportedOperationException(errMsg) - } - case _ => in - } - } - case e: Expression => e - } - } - } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala index fab973ffb0ed..8ed2137f4489 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala @@ -16,14 +16,15 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.extension.columnar.transition.ColumnarToRowLike +import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, Transitions} import org.apache.gluten.utils.{LogLevelUtil, PlanUtil} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.{PlanChangeLogger, Rule} import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive.BroadcastQueryStageExec -import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ShuffleExchangeLike} +import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, BroadcastQueryStageExec} +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, ShuffleExchangeLike} +import org.apache.spark.sql.internal.SQLConf object MiscColumnarRules { object TransformPreOverrides { @@ -58,6 +59,101 @@ object MiscColumnarRules { } } + // Replaces all SubqueryBroadcastExec used by sub-queries with ColumnarSubqueryBroadcastExec. + // This prevents query execution from being failed by fallen-back SubqueryBroadcastExec with + // child plan with columnar output (e.g., an adaptive Spark plan that yields final plan that + // is full-offloaded). ColumnarSubqueryBroadcastExec is both compatible with row-based and + // columnar child plan so is always functional. + case class RewriteSubqueryBroadcast() extends Rule[SparkPlan] { + override def apply(plan: SparkPlan): SparkPlan = { + val out = plan.transformWithSubqueries { + case p => + // Since https://github.com/apache/incubator-gluten/pull/1851. + // + // When AQE is on, the AQE sub-query cache should already be filled with + // row-based SubqueryBroadcastExec for reusing. Thus we are doing the same + // memorize-and-reuse work here for the replaced columnar version. + val reuseRemoved = removeReuses(p) + val replaced = replace(reuseRemoved) + replaced + } + out + } + + private def removeReuses(p: SparkPlan): SparkPlan = { + val out = p.transformExpressions { + case pe: ExecSubqueryExpression => + val newPlan = pe.plan match { + case ReusedSubqueryExec(s: SubqueryBroadcastExec) => + // Remove ReusedSubqueryExec. We will re-create reuses in subsequent method + // #replace. + // + // We assume only meeting reused sub-queries in AQE execution. When AQE is off, + // Spark adds reuses only after applying columnar rules by preparation rule + // ReuseExchangeAndSubquery. + assert(s.child.isInstanceOf[AdaptiveSparkPlanExec]) + s + case other => + other + } + pe.withNewPlan(newPlan) + } + out + } + + private def replace(p: SparkPlan): SparkPlan = { + val out = p.transformExpressions { + case pe: ExecSubqueryExpression => + val newPlan = pe.plan match { + case s: SubqueryBroadcastExec => + val columnarSubqueryBroadcast = toColumnarSubqueryBroadcast(s) + val maybeReused = columnarSubqueryBroadcast.child match { + case a: AdaptiveSparkPlanExec if SQLConf.get.subqueryReuseEnabled => + val cached = a.context.subqueryCache.get(columnarSubqueryBroadcast.canonicalized) + if (cached.nonEmpty) { + // Reuse the one in cache. + ReusedSubqueryExec(cached.get) + } else { + // Place columnar sub-query broadcast into cache, then return it. + a.context.subqueryCache + .update(columnarSubqueryBroadcast.canonicalized, columnarSubqueryBroadcast) + columnarSubqueryBroadcast + } + case _ => + // We are not in AQE. + columnarSubqueryBroadcast + } + maybeReused + case other => other + } + pe.withNewPlan(newPlan) + } + out + } + + private def toColumnarBroadcastExchange( + exchange: BroadcastExchangeExec): ColumnarBroadcastExchangeExec = { + val newChild = Transitions.toBackendBatchPlan(exchange.child) + ColumnarBroadcastExchangeExec(exchange.mode, newChild) + } + + private def toColumnarSubqueryBroadcast( + from: SubqueryBroadcastExec): ColumnarSubqueryBroadcastExec = { + val newChild = from.child match { + case exchange: BroadcastExchangeExec => + toColumnarBroadcastExchange(exchange) + case aqe: AdaptiveSparkPlanExec => + // Keeps the child if its is AQE even if its supportsColumnar == false. + // ColumnarSubqueryBroadcastExec is compatible with both row-based + // and columnar inputs. + aqe + case other => other + } + val out = ColumnarSubqueryBroadcastExec(from.name, from.index, from.buildKeys, newChild) + out + } + } + // Remove topmost columnar-to-row otherwise AQE throws error. // See: org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec#newQueryStage // diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 6e4d37f633eb..8cd2a5fb67bd 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -20,18 +20,16 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ -import org.apache.gluten.expression.ExpressionConverter import org.apache.gluten.extension.GlutenPlan import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.{LogLevelUtil, PlanUtil} import org.apache.spark.api.python.EvalPythonExecTransformer import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} import org.apache.spark.sql.execution.datasources.WriteFilesExec -import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, FileScan} +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins._ import org.apache.spark.sql.execution.python.{ArrowEvalPythonExec, BatchEvalPythonExec} @@ -254,17 +252,7 @@ object OffloadOthers { def doReplace(p: SparkPlan): SparkPlan = { val plan = p if (TransformHints.isNotTransformable(plan)) { - logDebug(s"Columnar Processing for ${plan.getClass} is under row guard.") - plan match { - case plan: BatchScanExec => - return applyScanNotTransformable(plan) - case plan: FileSourceScanExec => - return applyScanNotTransformable(plan) - case plan if HiveTableScanExecTransformer.isHiveTableScan(plan) => - return applyScanNotTransformable(plan) - case p => - return p - } + return plan } plan match { case plan: BatchScanExec => @@ -404,44 +392,6 @@ object OffloadOthers { } } - // Since https://github.com/apache/incubator-gluten/pull/2701 - private def applyScanNotTransformable(plan: SparkPlan): SparkPlan = plan match { - case plan: FileSourceScanExec => - val newPartitionFilters = - ExpressionConverter.transformDynamicPruningExpr(plan.partitionFilters) - val newSource = plan.copy(partitionFilters = newPartitionFilters) - if (plan.logicalLink.nonEmpty) { - newSource.setLogicalLink(plan.logicalLink.get) - } - TransformHints.tag(newSource, TransformHints.getHint(plan)) - newSource - case plan: BatchScanExec => - val newPartitionFilters: Seq[Expression] = plan.scan match { - case scan: FileScan => - ExpressionConverter.transformDynamicPruningExpr(scan.partitionFilters) - case _ => - ExpressionConverter.transformDynamicPruningExpr(plan.runtimeFilters) - } - val newSource = plan.copy(runtimeFilters = newPartitionFilters) - if (plan.logicalLink.nonEmpty) { - newSource.setLogicalLink(plan.logicalLink.get) - } - TransformHints.tag(newSource, TransformHints.getHint(plan)) - newSource - case plan if HiveTableScanExecTransformer.isHiveTableScan(plan) => - val newPartitionFilters: Seq[Expression] = - ExpressionConverter.transformDynamicPruningExpr( - HiveTableScanExecTransformer.getPartitionFilters(plan)) - val newSource = HiveTableScanExecTransformer.copyWith(plan, newPartitionFilters) - if (plan.logicalLink.nonEmpty) { - newSource.setLogicalLink(plan.logicalLink.get) - } - TransformHints.tag(newSource, TransformHints.getHint(plan)) - newSource - case other => - throw new UnsupportedOperationException(s"${other.getClass.toString} is not supported.") - } - /** * Apply scan transformer for file source and batch source, * 1. create new filter and scan transformer, 2. validate, tag new scan as unsupported if @@ -456,18 +406,13 @@ object OffloadOthers { transformer } else { logDebug(s"Columnar Processing for ${plan.getClass} is currently unsupported.") - val newSource = plan.copy(partitionFilters = transformer.getPartitionFilters()) - TransformHints.tagNotTransformable(newSource, validationResult.reason.get) - newSource + TransformHints.tagNotTransformable(plan, validationResult.reason.get) + plan } case plan: BatchScanExec => ScanTransformerFactory.createBatchScanTransformer(plan) - case plan if HiveTableScanExecTransformer.isHiveTableScan(plan) => // TODO: Add DynamicPartitionPruningHiveScanSuite.scala - val newPartitionFilters: Seq[Expression] = - ExpressionConverter.transformDynamicPruningExpr( - HiveTableScanExecTransformer.getPartitionFilters(plan)) val hiveTableScanExecTransformer = BackendsApiManager.getSparkPlanExecApiInstance.genHiveTableScanExecTransformer(plan) val validateResult = hiveTableScanExecTransformer.doValidate() @@ -476,9 +421,8 @@ object OffloadOthers { return hiveTableScanExecTransformer } logDebug(s"Columnar Processing for ${plan.getClass} is currently unsupported.") - val newSource = HiveTableScanExecTransformer.copyWith(plan, newPartitionFilters) - TransformHints.tagNotTransformable(newSource, validateResult.reason.get) - newSource + TransformHints.tagNotTransformable(plan, validateResult.reason.get) + plan case other => throw new GlutenNotSupportException(s"${other.getClass.toString} is not supported.") } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index ca35c74f6892..d32cf2d22eb4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -372,7 +372,7 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { // If filter expressions aren't empty, we need to transform the inner operators. if (plan.partitionFilters.isEmpty) { val transformer = - ScanTransformerFactory.createFileSourceScanTransformer(plan, validation = true) + ScanTransformerFactory.createFileSourceScanTransformer(plan) transformer.doValidate().tagOnFallback(plan) } case plan if HiveTableScanExecTransformer.isHiveTableScan(plan) => diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index a259641f5049..26201dc1baa3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -19,7 +19,7 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ -import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow} +import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, RewriteSubqueryBroadcast} import org.apache.gluten.extension.columnar.transition.{InsertTransitions, RemoveTransitions} import org.apache.gluten.extension.columnar.util.AdaptiveContext import org.apache.gluten.metrics.GlutenTimeMetric @@ -101,7 +101,8 @@ class EnumeratedApplier(session: SparkSession) (_: SparkSession) => RemoveTransitions, (spark: SparkSession) => FallbackOnANSIMode(spark), (spark: SparkSession) => PlanOneRowRelation(spark), - (_: SparkSession) => FallbackEmptySchemaRelation() + (_: SparkSession) => FallbackEmptySchemaRelation(), + (_: SparkSession) => RewriteSubqueryBroadcast() ) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarValidationRules() ::: List((spark: SparkSession) => MergeTwoPhasesHashBaseAggregate(spark)) ::: diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index 941677a6b933..eb5c561bfa8d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -19,7 +19,7 @@ package org.apache.gluten.extension.columnar.heuristic import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ -import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPreOverrides} +import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, RewriteSubqueryBroadcast, TransformPreOverrides} import org.apache.gluten.extension.columnar.rewrite.RewriteSparkPlanRulesManager import org.apache.gluten.extension.columnar.transition.{InsertTransitions, RemoveTransitions} import org.apache.gluten.extension.columnar.util.AdaptiveContext @@ -112,7 +112,8 @@ class HeuristicApplier(session: SparkSession) (spark: SparkSession) => FallbackOnANSIMode(spark), (spark: SparkSession) => FallbackMultiCodegens(spark), (spark: SparkSession) => PlanOneRowRelation(spark), - (_: SparkSession) => FallbackEmptySchemaRelation() + (_: SparkSession) => FallbackEmptySchemaRelation(), + (_: SparkSession) => RewriteSubqueryBroadcast() ) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarValidationRules() ::: List( diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala index a74d428fd452..2c1edd04bb4a 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarSubqueryBroadcastExec.scala @@ -24,8 +24,6 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec -import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.execution.joins.{BuildSideRelation, HashedRelation, HashJoin, LongHashedRelation} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.util.ThreadUtils @@ -75,35 +73,25 @@ case class ColumnarSubqueryBroadcastExec( SQLExecution.withExecutionId(session, executionId) { val rows = GlutenTimeMetric.millis(longMetric("collectTime")) { _ => - val exchangeChild = child match { - case exec: ReusedExchangeExec => - exec.child - case _ => - child - } - if ( - exchangeChild.isInstanceOf[ColumnarBroadcastExchangeExec] || - exchangeChild.isInstanceOf[AdaptiveSparkPlanExec] - ) { - // transform broadcasted columnar value to Array[InternalRow] by key - exchangeChild - .executeBroadcast[BuildSideRelation] - .value - .transform(buildKeys(index)) - .distinct - } else { - val broadcastRelation = exchangeChild.executeBroadcast[HashedRelation]().value - val (iter, expr) = if (broadcastRelation.isInstanceOf[LongHashedRelation]) { - (broadcastRelation.keys(), HashJoin.extractKeyExprAt(buildKeys, index)) - } else { - ( - broadcastRelation.keys(), - BoundReference(index, buildKeys(index).dataType, buildKeys(index).nullable)) - } - - val proj = UnsafeProjection.create(expr) - val keyIter = iter.map(proj).map(_.copy()) - keyIter.toArray[InternalRow].distinct + val relation = child.executeBroadcast[Any]().value + relation match { + case b: BuildSideRelation => + // Transform columnar broadcast value to Array[InternalRow] by key. + b.transform(buildKeys(index)).distinct + case h: HashedRelation => + val (iter, expr) = if (h.isInstanceOf[LongHashedRelation]) { + (h.keys(), HashJoin.extractKeyExprAt(buildKeys, index)) + } else { + ( + h.keys(), + BoundReference(index, buildKeys(index).dataType, buildKeys(index).nullable)) + } + val proj = UnsafeProjection.create(expr) + val keyIter = iter.map(proj).map(_.copy()) + keyIter.toArray[InternalRow].distinct + case other => + throw new UnsupportedOperationException( + s"Unrecognizable broadcast relation: $other") } } val dataSize = rows.map(_.asInstanceOf[UnsafeRow].getSizeInBytes).sum diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala b/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala index 9e97a3687656..1cd735cf7ee7 100644 --- a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala +++ b/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala @@ -82,14 +82,12 @@ case class DeltaScanTransformer( object DeltaScanTransformer { - def apply( - scanExec: FileSourceScanExec, - newPartitionFilters: Seq[Expression]): DeltaScanTransformer = { + def apply(scanExec: FileSourceScanExec): DeltaScanTransformer = { new DeltaScanTransformer( scanExec.relation, scanExec.output, scanExec.requiredSchema, - newPartitionFilters, + scanExec.partitionFilters, scanExec.optionalBucketSet, scanExec.optionalNumCoalescedBuckets, scanExec.dataFilters, diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala b/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala index a7cecde7c0db..e482150b8e29 100644 --- a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala +++ b/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala @@ -16,7 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.FileSourceScanExec class DeltaScanTransformerProvider extends DataSourceScanTransformerRegister { @@ -24,8 +23,7 @@ class DeltaScanTransformerProvider extends DataSourceScanTransformerRegister { override val scanClassName: String = "org.apache.spark.sql.delta.DeltaParquetFileFormat" override def createDataSourceTransformer( - batchScan: FileSourceScanExec, - newPartitionFilters: Seq[Expression]): FileSourceScanExecTransformerBase = { - DeltaScanTransformer(batchScan, newPartitionFilters) + batchScan: FileSourceScanExec): FileSourceScanExecTransformerBase = { + DeltaScanTransformer(batchScan) } } diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala index 6e079bf7e10a..5a735b802adb 100644 --- a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala +++ b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala @@ -80,13 +80,11 @@ case class IcebergScanTransformer( } object IcebergScanTransformer { - def apply( - batchScan: BatchScanExec, - newPartitionFilters: Seq[Expression]): IcebergScanTransformer = { + def apply(batchScan: BatchScanExec): IcebergScanTransformer = { new IcebergScanTransformer( batchScan.output, batchScan.scan, - newPartitionFilters, + batchScan.runtimeFilters, table = SparkShimLoader.getSparkShims.getBatchScanExecTable(batchScan), keyGroupedPartitioning = SparkShimLoader.getSparkShims.getKeyGroupedPartitioning(batchScan), commonPartitionValues = SparkShimLoader.getSparkShims.getCommonPartitionValues(batchScan) diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergTransformerProvider.scala b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergTransformerProvider.scala index 1ebeebf00be0..dc521f39c1b9 100644 --- a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergTransformerProvider.scala +++ b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergTransformerProvider.scala @@ -16,7 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.datasources.v2.BatchScanExec class IcebergTransformerProvider extends DataSourceScanTransformerRegister { @@ -24,8 +23,7 @@ class IcebergTransformerProvider extends DataSourceScanTransformerRegister { override val scanClassName: String = "org.apache.iceberg.spark.source.SparkBatchQueryScan" override def createDataSourceV2Transformer( - batchScan: BatchScanExec, - newPartitionFilters: Seq[Expression]): BatchScanExecTransformerBase = { - IcebergScanTransformer(batchScan, newPartitionFilters) + batchScan: BatchScanExec): BatchScanExecTransformerBase = { + IcebergScanTransformer(batchScan) } } From 7132b6d909987786840e592773183762aafb248a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Wed, 12 Jun 2024 10:40:54 +0800 Subject: [PATCH 247/402] [VL] Support Row Index Metadata Column (#5351) * inital row index change * test private velox branch * fix build * refactor parsePartitionAndMetadataColumns to parseColunTypes * fix ut * fix build * fix ci * fix clickhouse ci * refacor code * fix all ut of file metadata * small change * fix format * fix ci ut * small refactor * small change * address some remaining comments --------- Co-authored-by: Yangyang Gao --- .../clickhouse/CHTransformerApi.scala | 2 ++ .../gluten/utils/CHInputPartitionsUtil.scala | 2 ++ .../backendsapi/velox/VeloxBackend.scala | 2 ++ .../velox/VeloxTransformerApi.scala | 2 ++ cpp/velox/substrait/SubstraitParser.cc | 3 ++ .../substrait/proto/substrait/type.proto | 1 + .../backendsapi/BackendSettingsApi.scala | 2 ++ .../gluten/backendsapi/SparkPlanExecApi.scala | 16 ++++++---- .../gluten/backendsapi/TransformerApi.scala | 3 +- .../execution/BasicScanExecTransformer.scala | 3 ++ .../execution/BatchScanExecTransformer.scala | 8 +++++ .../FileSourceScanExecTransformer.scala | 12 +++++++- .../gluten/utils/InputPartitionsUtil.scala | 4 ++- .../TestFileSourceScanExecTransformer.scala | 1 + .../TestFileSourceScanExecTransformer.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 5 +--- .../GlutenFileMetadataStructSuite.scala | 1 + .../parquet/GlutenParquetRowIndexSuite.scala | 4 --- .../TestFileSourceScanExecTransformer.scala | 1 + .../utils/velox/VeloxTestSettings.scala | 3 -- .../GlutenFileMetadataStructSuite.scala | 3 +- .../parquet/GlutenParquetRowIndexSuite.scala | 2 -- .../TestFileSourceScanExecTransformer.scala | 1 + .../apache/gluten/sql/shims/SparkShims.scala | 9 ++++-- .../sql/shims/spark32/Spark32Shims.scala | 7 +++++ .../sql/shims/spark33/Spark33Shims.scala | 7 +++++ .../sql/shims/spark34/Spark34Shims.scala | 29 ++++++++++++++++++- .../execution/FileSourceScanExecShim.scala | 7 +---- .../datasources/v2/BatchScanExecShim.scala | 6 +--- .../sql/shims/spark35/Spark35Shims.scala | 29 +++++++++++++++++-- .../execution/FileSourceScanExecShim.scala | 6 +--- .../datasources/v2/BatchScanExecShim.scala | 6 +--- 32 files changed, 137 insertions(+), 51 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index a010a986cd4c..9653256256bd 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -45,6 +45,7 @@ class CHTransformerApi extends TransformerApi with Logging { /** Generate Seq[InputPartition] for FileSourceScanExecTransformer. */ def genInputPartitionSeq( relation: HadoopFsRelation, + requiredSchema: StructType, selectedPartitions: Array[PartitionDirectory], output: Seq[Attribute], bucketedScan: Boolean, @@ -73,6 +74,7 @@ class CHTransformerApi extends TransformerApi with Logging { // Generate FilePartition for Parquet CHInputPartitionsUtil( relation, + requiredSchema, selectedPartitions, output, bucketedScan, diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala index 90ab336689fe..c808abf3d2ce 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.PartitionedFileUtil import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.types.StructType import org.apache.spark.util.SparkResourceUtil import org.apache.spark.util.collection.BitSet @@ -33,6 +34,7 @@ import scala.collection.mutable.ArrayBuffer case class CHInputPartitionsUtil( relation: HadoopFsRelation, + requiredSchema: StructType, selectedPartitions: Array[PartitionDirectory], output: Seq[Attribute], bucketedScan: Boolean, diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 21e6246d1271..060a0c4cbd34 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -271,6 +271,8 @@ object VeloxBackendSettings extends BackendSettingsApi { override def supportNativeMetadataColumns(): Boolean = true + override def supportNativeRowIndexColumn(): Boolean = true + override def supportExpandExec(): Boolean = true override def supportSortExec(): Boolean = true diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index aadfcd9b7d1e..ac24b53af11b 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -39,6 +39,7 @@ class VeloxTransformerApi extends TransformerApi with Logging { /** Generate Seq[InputPartition] for FileSourceScanExecTransformer. */ def genInputPartitionSeq( relation: HadoopFsRelation, + requiredSchema: StructType, selectedPartitions: Array[PartitionDirectory], output: Seq[Attribute], bucketedScan: Boolean, @@ -48,6 +49,7 @@ class VeloxTransformerApi extends TransformerApi with Logging { filterExprs: Seq[Expression] = Seq.empty): Seq[InputPartition] = { InputPartitionsUtil( relation, + requiredSchema, selectedPartitions, output, bucketedScan, diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 0880f3e3d915..5555ecfef954 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -131,6 +131,9 @@ void SubstraitParser::parseColumnTypes( case ::substrait::NamedStruct::METADATA_COL: columnTypes.push_back(ColumnType::kSynthesized); break; + case ::substrait::NamedStruct::ROWINDEX_COL: + columnTypes.push_back(ColumnType::kRowIndex); + break; default: VELOX_FAIL("Unspecified column type."); } diff --git a/gluten-core/src/main/resources/substrait/proto/substrait/type.proto b/gluten-core/src/main/resources/substrait/proto/substrait/type.proto index 5c7ee6a382ce..b5fcb95623ac 100644 --- a/gluten-core/src/main/resources/substrait/proto/substrait/type.proto +++ b/gluten-core/src/main/resources/substrait/proto/substrait/type.proto @@ -238,5 +238,6 @@ message NamedStruct { NORMAL_COL = 0; PARTITION_COL = 1; METADATA_COL = 2; + ROWINDEX_COL = 3; } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index b7a3bc1b6ef2..b132366e6e1d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -43,6 +43,8 @@ trait BackendSettingsApi { options: Map[String, String]): ValidationResult = ValidationResult.ok def supportNativeWrite(fields: Array[StructField]): Boolean = true def supportNativeMetadataColumns(): Boolean = false + def supportNativeRowIndexColumn(): Boolean = false + def supportExpandExec(): Boolean = false def supportSortExec(): Boolean = false def supportSortMergeJoinExec(): Boolean = true diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 8bc8e136bd5d..2b4255db4fc2 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -20,6 +20,7 @@ import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ import org.apache.gluten.extension.columnar.transition.{Convention, ConventionFunc} +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode, WindowFunctionNode} import org.apache.spark.ShuffleDependency @@ -661,17 +662,20 @@ trait SparkPlanExecApi { def postProcessPushDownFilter( extraFilters: Seq[Expression], sparkExecNode: LeafExecNode): Seq[Expression] = { + def getPushedFilter(dataFilters: Seq[Expression]): Seq[Expression] = { + val pushedFilters = + dataFilters ++ FilterHandler.getRemainingFilters(dataFilters, extraFilters) + pushedFilters.filterNot(_.references.exists { + attr => SparkShimLoader.getSparkShims.isRowIndexMetadataColumn(attr.name) + }) + } sparkExecNode match { case fileSourceScan: FileSourceScanExec => - fileSourceScan.dataFilters ++ FilterHandler.getRemainingFilters( - fileSourceScan.dataFilters, - extraFilters) + getPushedFilter(fileSourceScan.dataFilters) case batchScan: BatchScanExec => batchScan.scan match { case fileScan: FileScan => - fileScan.dataFilters ++ FilterHandler.getRemainingFilters( - fileScan.dataFilters, - extraFilters) + getPushedFilter(fileScan.dataFilters) case _ => // TODO: For data lake format use pushedFilters in SupportsPushDownFilters extraFilters diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala index 05a639ac2dc7..522be378790b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/TransformerApi.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.types.{DataType, DecimalType} +import org.apache.spark.sql.types.{DataType, DecimalType, StructType} import org.apache.spark.util.collection.BitSet import com.google.protobuf.{Any, Message} @@ -34,6 +34,7 @@ trait TransformerApi { /** Generate Seq[InputPartition] for FileSourceScanExecTransformer. */ def genInputPartitionSeq( relation: HadoopFsRelation, + requiredSchema: StructType, selectedPartitions: Array[PartitionDirectory], output: Seq[Attribute], bucketedScan: Boolean, diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index af35957ec393..3bbd99c50a6a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -19,6 +19,7 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.expression.{ConverterUtils, ExpressionConverter} import org.apache.gluten.extension.ValidationResult +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.`type`.ColumnTypeNode import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.extensions.ExtensionBuilder @@ -115,6 +116,8 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource attr => if (getPartitionSchema.exists(_.name.equals(attr.name))) { new ColumnTypeNode(1) + } else if (SparkShimLoader.getSparkShims.isRowIndexMetadataColumn(attr.name)) { + new ColumnTypeNode(3) } else if (attr.isMetadataCol) { new ColumnTypeNode(2) } else { diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala index 64d9d6546bd8..6bff68895a24 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BatchScanExecTransformer.scala @@ -20,6 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.spark.sql.catalyst.InternalRow @@ -135,6 +136,13 @@ abstract class BatchScanExecTransformerBase( return ValidationResult.notOk(s"Unsupported aggregation push down for $scan.") } + if ( + SparkShimLoader.getSparkShims.findRowIndexColumnIndexInSchema(schema) > 0 && + !BackendsApiManager.getSettings.supportNativeRowIndexColumn() + ) { + return ValidationResult.notOk("Unsupported row index column scan in native.") + } + if (hasUnsupportedColumns) { return ValidationResult.notOk(s"Unsupported columns scan in native.") } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala index f4855840256b..c3c296c13c3c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala @@ -19,6 +19,7 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.spark.sql.catalyst.TableIdentifier @@ -108,13 +109,15 @@ abstract class FileSourceScanExecTransformerBase( override def getPartitions: Seq[InputPartition] = { BackendsApiManager.getTransformerApiInstance.genInputPartitionSeq( relation, + requiredSchema, dynamicallySelectedPartitions, output, bucketedScan, optionalBucketSet, optionalNumCoalescedBuckets, disableBucketedScan, - filterExprs()) + filterExprs() + ) } override def getPartitionSchema: StructType = relation.partitionSchema @@ -132,6 +135,13 @@ abstract class FileSourceScanExecTransformerBase( return ValidationResult.notOk(s"Unsupported metadata columns scan in native.") } + if ( + SparkShimLoader.getSparkShims.findRowIndexColumnIndexInSchema(schema) > 0 && + !BackendsApiManager.getSettings.supportNativeRowIndexColumn() + ) { + return ValidationResult.notOk("Unsupported row index column scan in native.") + } + if (hasUnsupportedColumns) { return ValidationResult.notOk(s"Unsupported columns scan in native.") } diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/InputPartitionsUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/InputPartitionsUtil.scala index 135721ff03a7..fa0823e1c6f1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/InputPartitionsUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/InputPartitionsUtil.scala @@ -22,10 +22,12 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.datasources.{FilePartition, HadoopFsRelation, PartitionDirectory} +import org.apache.spark.sql.types.StructType import org.apache.spark.util.collection.BitSet case class InputPartitionsUtil( relation: HadoopFsRelation, + requiredSchema: StructType, selectedPartitions: Array[PartitionDirectory], output: Seq[Attribute], bucketedScan: Boolean, @@ -58,7 +60,7 @@ case class InputPartitionsUtil( // getPath() is very expensive so we only want to call it once in this block: val filePath = file.getPath val isSplitable = - relation.fileFormat.isSplitable(relation.sparkSession, relation.options, filePath) + SparkShimLoader.getSparkShims.isFileSplittable(relation, filePath, requiredSchema) SparkShimLoader.getSparkShims.splitFiles( sparkSession = relation.sparkSession, file = file, diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala index a7ad96600d8f..88d771ec8a4f 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala @@ -52,6 +52,7 @@ case class TestFileSourceScanExecTransformer( override def getPartitions: Seq[InputPartition] = BackendsApiManager.getTransformerApiInstance.genInputPartitionSeq( relation, + requiredSchema, selectedPartitions, output, bucketedScan, diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala index 592019f34ccf..85af9a623853 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala @@ -50,6 +50,7 @@ case class TestFileSourceScanExecTransformer( override def getPartitions: Seq[InputPartition] = BackendsApiManager.getTransformerApiInstance.genInputPartitionSeq( relation, + requiredSchema, selectedPartitions, output, bucketedScan, diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index c532355df6de..060f199d1e23 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -764,6 +764,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenFileFormatWriterSuite] enableSuite[GlutenFileIndexSuite] enableSuite[GlutenFileMetadataStructSuite] + enableSuite[GlutenFileMetadataStructRowIndexSuite] enableSuite[GlutenParquetV1AggregatePushDownSuite] enableSuite[GlutenParquetV2AggregatePushDownSuite] enableSuite[GlutenOrcV1AggregatePushDownSuite] @@ -1191,10 +1192,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenResolveDefaultColumnsSuite] enableSuite[GlutenSubqueryHintPropagationSuite] enableSuite[GlutenUrlFunctionsSuite] - enableSuite[GlutenFileMetadataStructRowIndexSuite] - // Row index metadata column support in Velox isn't ready yet, refer velox-9147 - .exclude("reading _tmp_metadata_row_index - not present in a table") - .exclude("reading _tmp_metadata_row_index - present in a table") enableSuite[GlutenParquetRowIndexSuite] .excludeByPrefix("row index generation") .excludeByPrefix("invalid row index column type") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index 5e4d1ed8f81f..1d35c8656de1 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -46,6 +46,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS private val METADATA_FILE_NAME = "_metadata.file_name" private val METADATA_FILE_SIZE = "_metadata.file_size" private val METADATA_FILE_MODIFICATION_TIME = "_metadata.file_modification_time" + private val FILE_FORMAT = "fileFormat" private def getMetadataForFile(f: File): Map[String, Any] = { Map( diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala index 6f153450cb96..0113a92282fc 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -39,10 +39,6 @@ import scala.collection.JavaConverters._ class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTestsBaseTrait { import testImplicits._ - override def beforeAll(): Unit = { - super.beforeAll() - sparkContext.setLogLevel("info") - } private def readRowGroupRowCounts(path: String): Seq[Long] = { ParquetFooterReader diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala index 592019f34ccf..85af9a623853 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala @@ -50,6 +50,7 @@ case class TestFileSourceScanExecTransformer( override def getPartitions: Seq[InputPartition] = BackendsApiManager.getTransformerApiInstance.genInputPartitionSeq( relation, + requiredSchema, selectedPartitions, output, bucketedScan, diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 104c22dbe482..2911512f5512 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1224,9 +1224,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenGroupBasedMergeIntoTableSuite] enableSuite[GlutenFileSourceCustomMetadataStructSuite] enableSuite[GlutenParquetFileMetadataStructRowIndexSuite] - // Row index metadata column support in Velox isn't ready yet, refer velox-9147 - .exclude("reading _tmp_metadata_row_index - not present in a table") - .exclude("reading _tmp_metadata_row_index - present in a table") enableSuite[GlutenTableLocationSuite] enableSuite[GlutenRemoveRedundantWindowGroupLimitsSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index 5e4d1ed8f81f..efa0fbae062b 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -21,7 +21,6 @@ import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType} @@ -60,7 +59,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS f: (DataFrame, Map[String, Any], Map[String, Any]) => Unit): Unit = { Seq("parquet").foreach { testFileFormat => - test(s"$GLUTEN_TEST metadata struct ($testFileFormat): " + testName) { + testGluten(s"metadata struct ($testFileFormat): " + testName) { withTempDir { dir => import scala.collection.JavaConverters._ diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala index abf21651f827..ad41a8395fd0 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -221,8 +221,6 @@ class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTest df.schema.add(rowIndexColName, LongType, nullable = true) } - logInfo(s"gyytest schemaWithRowIndex $schemaWithRowIdx") - df.write .format(conf.writeFormat) .option(ParquetOutputFormat.BLOCK_SIZE, conf.rowGroupSize) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala index 592019f34ccf..85af9a623853 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/TestFileSourceScanExecTransformer.scala @@ -50,6 +50,7 @@ case class TestFileSourceScanExecTransformer( override def getPartitions: Seq[InputPartition] = BackendsApiManager.getTransformerApiInstance.genInputPartitionSeq( relation, + requiredSchema, selectedPartitions, output, bucketedScan, diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala index 8bbc6d3d18d4..d9d356b67ada 100644 --- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala @@ -40,11 +40,12 @@ import org.apache.spark.sql.connector.read.{InputPartition, Scan} import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, GlobalLimitExec, SparkPlan, TakeOrderedAndProjectExec} import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec import org.apache.spark.sql.execution.command.DataWritingCommandExec -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionDirectory, PartitionedFile, PartitioningAwareFileIndex, WriteJobDescription, WriteTaskResult} +import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, HadoopFsRelation, PartitionDirectory, PartitionedFile, PartitioningAwareFileIndex, WriteJobDescription, WriteTaskResult} +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ShuffleExchangeLike} -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} @@ -193,8 +194,12 @@ trait SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] + def isFileSplittable(relation: HadoopFsRelation, filePath: Path, sparkSchema: StructType): Boolean + def isRowIndexMetadataColumn(name: String): Boolean + def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala index f24aef66a1cb..22122c5837dc 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala @@ -189,8 +189,15 @@ class Spark32Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files + def isFileSplittable( + relation: HadoopFsRelation, + filePath: Path, + sparkSchema: StructType): Boolean = true + def isRowIndexMetadataColumn(name: String): Boolean = false + def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = -1 + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala index 68fc4ad0dc1a..fdc782484e02 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala @@ -278,8 +278,15 @@ class Spark33Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files + def isFileSplittable( + relation: HadoopFsRelation, + filePath: Path, + sparkSchema: StructType): Boolean = true + def isRowIndexMetadataColumn(name: String): Boolean = false + def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = -1 + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 7d9fc389b7cb..171d412389be 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -47,7 +47,7 @@ import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} @@ -332,10 +332,37 @@ class Spark34Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files + def isFileSplittable( + relation: HadoopFsRelation, + filePath: Path, + sparkSchema: StructType): Boolean = { + // SPARK-39634: Allow file splitting in combination with row index generation once + // the fix for PARQUET-2161 is available. + relation.fileFormat + .isSplitable(relation.sparkSession, relation.options, filePath) && + !(RowIndexUtil.findRowIndexColumnIndexInSchema(sparkSchema) >= 0) + } + def isRowIndexMetadataColumn(name: String): Boolean = { name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME } + def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = { + sparkSchema.fields.zipWithIndex.find { + case (field: StructField, _: Int) => + field.name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + } match { + case Some((field: StructField, idx: Int)) => + if (field.dataType != LongType && field.dataType != IntegerType) { + throw new RuntimeException( + s"${FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME} " + + s"must be of LongType or IntegerType") + } + idx + case _ => -1 + } + } + def splitFiles( sparkSession: SparkSession, file: FileStatus, diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index 33df953f32c8..bc3893ca201a 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -63,16 +63,11 @@ abstract class FileSourceScanExecShim( def hasUnsupportedColumns: Boolean = { val metadataColumnsNames = metadataColumns.map(_.name) - // row_index metadata is not support yet - metadataColumnsNames.contains(FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) || output .filterNot(metadataColumns.toSet) .exists(v => metadataColumnsNames.contains(v.name)) || // Below name has special meaning in Velox. - output.exists( - a => - a.name == "$path" || a.name == "$bucket" || - a.name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) + output.exists(a => a.name == "$path" || a.name == "$bucket") } def isMetadataColumn(attr: Attribute): Boolean = metadataColumns.contains(attr) diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 64afc8193f4e..bd3b09a01ef4 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -65,14 +65,10 @@ abstract class BatchScanExecShim( // TODO, fallback if user define same name column due to we can't right now // detect which column is metadata column which is user defined column. val metadataColumnsNames = metadataColumns.map(_.name) - metadataColumnsNames.contains(FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) || output .filterNot(metadataColumns.toSet) .exists(v => metadataColumnsNames.contains(v.name)) || - output.exists( - a => - a.name == "$path" || a.name == "$bucket" || - a.name == FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) + output.exists(a => a.name == "$path" || a.name == "$bucket") } override def doExecuteColumnar(): RDD[ColumnarBatch] = { diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 54cea6993d13..142403ada099 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -42,14 +42,14 @@ import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Sca import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetRowIndexUtil} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ShuffleExchangeLike} import org.apache.spark.sql.execution.window.{WindowGroupLimitExec, WindowGroupLimitExecShim} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} @@ -361,8 +361,31 @@ class Spark35Shims extends SparkShims { def getFileStatus(partition: PartitionDirectory): Seq[FileStatus] = partition.files.map(_.fileStatus) - def isRowIndexMetadataColumn(name: String): Boolean = { + def isFileSplittable( + relation: HadoopFsRelation, + filePath: Path, + sparkSchema: StructType): Boolean = { + relation.fileFormat + .isSplitable(relation.sparkSession, relation.options, filePath) + } + + def isRowIndexMetadataColumn(name: String): Boolean = name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + + def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = { + sparkSchema.fields.zipWithIndex.find { + case (field: StructField, _: Int) => + field.name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + } match { + case Some((field: StructField, idx: Int)) => + if (field.dataType != LongType && field.dataType != IntegerType) { + throw new RuntimeException( + s"${ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME} " + + "must be of LongType or IntegerType") + } + idx + case _ => -1 + } } def splitFiles( diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala index dccf1bbced1d..c8795e31ceb4 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/FileSourceScanExecShim.scala @@ -67,14 +67,10 @@ abstract class FileSourceScanExecShim( // TODO, fallback if user define same name column due to we can't right now // detect which column is metadata column which is user defined column. val metadataColumnsNames = metadataColumns.map(_.name) - metadataColumnsNames.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) || output .filterNot(metadataColumns.toSet) .exists(v => metadataColumnsNames.contains(v.name)) || - output.exists( - a => - a.name == "$path" || a.name == "$bucket" || - a.name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) + output.exists(a => a.name == "$path" || a.name == "$bucket") } def isMetadataColumn(attr: Attribute): Boolean = metadataColumns.contains(attr) diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala index 8949a46a1ddd..343070d7f209 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExecShim.scala @@ -67,14 +67,10 @@ abstract class BatchScanExecShim( // TODO, fallback if user define same name column due to we can't right now // detect which column is metadata column which is user defined column. val metadataColumnsNames = metadataColumns.map(_.name) - metadataColumnsNames.contains(ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) || output .filterNot(metadataColumns.toSet) .exists(v => metadataColumnsNames.contains(v.name)) || - output.exists( - a => - a.name == "$path" || a.name == "$bucket" || - a.name == ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME) + output.exists(a => a.name == "$path" || a.name == "$bucket") } override def doExecuteColumnar(): RDD[ColumnarBatch] = { From a4f09134acb2b9e82331fac9366d67e62b36b509 Mon Sep 17 00:00:00 2001 From: Yuan Date: Wed, 12 Jun 2024 10:53:59 +0800 Subject: [PATCH 248/402] [MISC] adding discussion link in issue template (#6047) misc change for adding discussion section in issue template Signed-off-by: Yuan Zhou --- .github/ISSUE_TEMPLATE/config.yml | 20 ++++++++++++++++++++ dev/info.sh | 1 + 2 files changed, 21 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..545ccc8c21d7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +blank_issues_enabled: false +contact_links: + - name: Gluten Discussions + url: https://github.com/apache/incubator-gluten/discussions + about: Ask questions or discuss new feature ideas here. diff --git a/dev/info.sh b/dev/info.sh index ed0e0a8dd390..8a5fbd182f84 100644 --- a/dev/info.sh +++ b/dev/info.sh @@ -38,6 +38,7 @@ Commit: $(git rev-parse HEAD 2> /dev/null || echo "Not in a git repo.") CMake Version: $(cmake --version | grep -oE '[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+') System: $(print_info 'CMAKE_SYSTEM "') Arch: $(print_info 'CMAKE_SYSTEM_PROCESSOR') +CPU Name: $(lscpu | grep 'Model name') C++ Compiler: $(print_info 'CMAKE_CXX_COMPILER ==') C++ Compiler Version: $(print_info 'CMAKE_CXX_COMPILER_VERSION') C Compiler: $(print_info 'CMAKE_C_COMPILER ==') From dab708026694c067b104ab901aab78bccee9463c Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:11:18 +0800 Subject: [PATCH 249/402] [VL] Support PreciseTimestampConversion function (#6036) [VL] Support PreciseTimestampConversion function. --- .../velox/VeloxSparkPlanExecApi.scala | 20 ++++++++++++++++++ .../ScalarFunctionsValidateSuite.scala | 21 +++++++++++++++++++ .../gluten/backendsapi/SparkPlanExecApi.scala | 7 +++++++ .../expression/ExpressionConverter.scala | 6 ++++++ .../expression/ExpressionMappings.scala | 1 + .../gluten/expression/ExpressionNames.scala | 1 + 6 files changed, 56 insertions(+) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 66ca8660a50c..26b4c508221d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -280,6 +280,26 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(endDate, startDate), original) } + override def genPreciseTimestampConversionTransformer( + substraitExprName: String, + children: Seq[ExpressionTransformer], + expr: PreciseTimestampConversion): ExpressionTransformer = { + // Expression used internally to convert the TimestampType to Long and back without losing + // precision, i.e. in microseconds. + val (newSubstraitName, newExpr) = expr match { + case _ @PreciseTimestampConversion(_, TimestampType, LongType) => + (ExpressionMappings.expressionsMap(classOf[UnixMicros]), UnixMicros(expr.child)) + case _ @PreciseTimestampConversion(_, LongType, TimestampType) => + ( + ExpressionMappings.expressionsMap(classOf[MicrosToTimestamp]), + MicrosToTimestamp(expr.child)) + case _ => + // TimestampNTZType is not supported here. + throw new GlutenNotSupportException("PreciseTimestampConversion is not supported") + } + GenericExpressionTransformer(newSubstraitName, children, newExpr) + } + /** * Generate FilterExecTransformer. * diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 6df3a062331f..a23fdf243888 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -1077,4 +1077,25 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } } + + test("PreciseTimestampConversion") { + withTempPath { + path => + val df = spark + .sql( + "select * from VALUES ('A1', TIMESTAMP'2021-01-01 00:00:00'), " + + "('A1', TIMESTAMP'2021-01-01 00:04:30'), ('A1', TIMESTAMP'2021-01-01 00:06:00'), " + + "('A2', TIMESTAMP'2021-01-01 00:01:00') AS tab(a, b)") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("string_timestamp") + + runQueryAndCompare( + "SELECT a, window.start, window.end, count(*) as cnt FROM" + + " string_timestamp GROUP by a, window(b, '5 minutes') ORDER BY a, start;") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 2b4255db4fc2..9a37c4a40dd1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -306,6 +306,13 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, children, expr) } + def genPreciseTimestampConversionTransformer( + substraitExprName: String, + children: Seq[ExpressionTransformer], + expr: PreciseTimestampConversion): ExpressionTransformer = { + throw new GlutenNotSupportException("PreciseTimestampConversion is not supported") + } + /** * Generate ShuffleDependency for ColumnarShuffleExchangeExec. * diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 5d0af9e526ee..464bbbfd002c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -650,6 +650,12 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(s.child, attributeSeq, expressionsMap), LiteralTransformer(Literal(s.randomSeed.get))), s) + case c: PreciseTimestampConversion => + BackendsApiManager.getSparkPlanExecApiInstance.genPreciseTimestampConversionTransformer( + substraitExprName, + Seq(replaceWithExpressionTransformerInternal(c.child, attributeSeq, expressionsMap)), + c + ) case expr => GenericExpressionTransformer( substraitExprName, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 1eade3da664a..230d91005e9c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -196,6 +196,7 @@ object ExpressionMappings { Sig[UnixMicros](UNIX_MICROS), Sig[MillisToTimestamp](TIMESTAMP_MILLIS), Sig[MicrosToTimestamp](TIMESTAMP_MICROS), + Sig[PreciseTimestampConversion](PRECYSE_TIMESTAMP_CONVERSION), // JSON functions Sig[GetJsonObject](GET_JSON_OBJECT), Sig[LengthOfJsonArray](JSON_ARRAY_LENGTH), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index dc98f31a395c..f817612a1e8d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -219,6 +219,7 @@ object ExpressionNames { final val UNIX_MICROS = "unix_micros" final val TIMESTAMP_MILLIS = "timestamp_millis" final val TIMESTAMP_MICROS = "timestamp_micros" + final val PRECYSE_TIMESTAMP_CONVERSION = "precise_timestamp_conversion" // JSON functions final val GET_JSON_OBJECT = "get_json_object" From 01de587a46e10b587c0634f36420e61f428ba01e Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:19:27 +0800 Subject: [PATCH 250/402] [VL] Daily Update Velox Version (2024_06_12) (#6051) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 4965dbac66cd..925a09630bb2 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_11 +VELOX_BRANCH=2024_06_12 VELOX_HOME="" #Set on run gluten on HDFS From 7f8d2330e12b75365219e92049b35da7f428bb62 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:26:25 +0800 Subject: [PATCH 251/402] [GLUTEN-6042][CH]Fix to_date function result type nullable check What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #6042) How was this patch tested? --- cpp-ch/local-engine/Functions/FunctionGetDateData.h | 5 +---- cpp-ch/local-engine/Functions/SparkFunctionToDateTime.h | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cpp-ch/local-engine/Functions/FunctionGetDateData.h b/cpp-ch/local-engine/Functions/FunctionGetDateData.h index 4f79d4bd0c4b..6cd93dd02c98 100644 --- a/cpp-ch/local-engine/Functions/FunctionGetDateData.h +++ b/cpp-ch/local-engine/Functions/FunctionGetDateData.h @@ -46,7 +46,7 @@ class FunctionGetDateData : public DB::IFunction FunctionGetDateData() = default; ~FunctionGetDateData() override = default; - DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t) const override + DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr &, size_t) const override { if (arguments.size() != 1) throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s arguments number must be 1.", getName()); @@ -54,9 +54,6 @@ class FunctionGetDateData : public DB::IFunction const DB::ColumnWithTypeAndName arg1 = arguments[0]; const auto * src_col = checkAndGetColumn(arg1.column.get()); size_t size = src_col->size(); - - if (!result_type->isNullable()) - throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s return type must be nullable", getName()); using ColVecTo = ColumnVector; typename ColVecTo::MutablePtr result_column = ColVecTo::create(size, 0); diff --git a/cpp-ch/local-engine/Functions/SparkFunctionToDateTime.h b/cpp-ch/local-engine/Functions/SparkFunctionToDateTime.h index d846f8956412..980af85bd983 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionToDateTime.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionToDateTime.h @@ -145,9 +145,6 @@ class SparkFunctionConvertToDateTime : public IFunction if (arguments.size() != 1 && arguments.size() != 2) throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s arguments number must be 1 or 2.", name); - if (!result_type->isNullable()) - throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s return type must be nullable", name); - if (!isDateTime64(removeNullable(result_type))) throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s return type must be datetime.", name); From 86ca8f7d5879fb332f24bfea3cb0b1f3b9493d8c Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 12 Jun 2024 15:06:57 +0800 Subject: [PATCH 252/402] fix (#6038) --- .../gluten/backendsapi/velox/VeloxListenerApi.scala | 9 ++++----- cpp/velox/udf/Udaf.h | 2 +- cpp/velox/udf/Udf.h | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 0b1d131d318e..0ad267d4c8d3 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -47,15 +47,13 @@ class VeloxListenerApi extends ListenerApi { StaticSQLConf.SPARK_CACHE_SERIALIZER.key, "org.apache.spark.sql.execution.ColumnarCachedBatchSerializer") } - UDFResolver.resolveUdfConf(conf, isDriver = true) - initialize(conf) + initialize(conf, isDriver = true) } override def onDriverShutdown(): Unit = shutdown() override def onExecutorStart(pc: PluginContext): Unit = { - UDFResolver.resolveUdfConf(pc.conf(), isDriver = false) - initialize(pc.conf()) + initialize(pc.conf(), isDriver = false) } override def onExecutorShutdown(): Unit = shutdown() @@ -158,8 +156,9 @@ class VeloxListenerApi extends ListenerApi { .commit() } - private def initialize(conf: SparkConf): Unit = { + private def initialize(conf: SparkConf, isDriver: Boolean): Unit = { SparkDirectoryUtil.init(conf) + UDFResolver.resolveUdfConf(conf, isDriver = isDriver) val debugJni = conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_MODE, defaultValue = false) && conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE, defaultValue = false) if (debugJni) { diff --git a/cpp/velox/udf/Udaf.h b/cpp/velox/udf/Udaf.h index 5b33e0611ba2..2f292fbc6cb3 100644 --- a/cpp/velox/udf/Udaf.h +++ b/cpp/velox/udf/Udaf.h @@ -23,7 +23,7 @@ struct UdafEntry { const char* name; const char* dataType; - size_t numArgs; + int numArgs; const char** argTypes; const char* intermediateType{nullptr}; diff --git a/cpp/velox/udf/Udf.h b/cpp/velox/udf/Udf.h index 1fa3c54d5213..a32bdaefe9ec 100644 --- a/cpp/velox/udf/Udf.h +++ b/cpp/velox/udf/Udf.h @@ -23,7 +23,7 @@ struct UdfEntry { const char* name; const char* dataType; - size_t numArgs; + int numArgs; const char** argTypes; bool variableArity{false}; From 31902aec7df4550baad93ddef91a55ade7ebeb84 Mon Sep 17 00:00:00 2001 From: Jacky Lee Date: Wed, 12 Jun 2024 16:57:52 +0800 Subject: [PATCH 253/402] [GLUTEN-5720][VL][FOLLOWUP] Fix invalid adding int to string (#6054) --- cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index 80509e055980..8e6dd7c1c31f 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -925,7 +925,7 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::JoinRel& joinRel case ::substrait::JoinRel_JoinType_JOIN_TYPE_ANTI: break; default: - LOG_VALIDATION_MSG("Join type is not supported: {}" + joinRel.type()); + LOG_VALIDATION_MSG("Join type is not supported: " + std::to_string(joinRel.type())); return false; } From 1c505dfd7cec48d04c37bbf714875883969cb1d7 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 12 Jun 2024 17:13:31 +0800 Subject: [PATCH 254/402] [VL] Gluten-it: Improve test report table format for parameterized test (#6052) --- .../integration/command/Parameterized.java | 2 +- .../integration/action/Parameterized.scala | 222 +++++++++++------- .../gluten/integration/action/Queries.scala | 6 +- .../integration/action/QueriesCompare.scala | 46 ++-- .../integration/action/TableRender.scala | 40 +++- .../gluten/integration/action/package.scala | 36 +++ .../integration/action/TableRenderTest.scala | 21 ++ 7 files changed, 243 insertions(+), 130 deletions(-) create mode 100644 tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java index 7e1234e7665d..cadff0a2db91 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java @@ -65,7 +65,7 @@ public class Parameterized implements Callable { @Override public Integer call() throws Exception { - final Map>>> parsed = new HashMap<>(); + final Map>>> parsed = new LinkedHashMap<>(); final Seq> excludedCombinations = JavaConverters.asScalaBufferConverter(Arrays.stream(excludedDims).map(d -> { final Matcher m = excludedDimsPattern.matcher(d); diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala index 8f5bc0946643..e2fc526ce566 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala @@ -18,12 +18,14 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.gluten.integration.action.Actions.QuerySelector +import org.apache.gluten.integration.action.TableRender.Field import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} import org.apache.spark.sql.ConfUtils.ConfImplicits._ import org.apache.spark.sql.SparkSessionSwitcher +import java.util.concurrent.atomic.AtomicInteger import scala.collection.mutable import scala.collection.mutable.ListBuffer @@ -39,6 +41,8 @@ class Parameterized( metrics: Array[String]) extends Action { + validateDims(configDimensions) + private def validateDims(configDimensions: Seq[Dim]): Unit = { if (configDimensions .map(dim => { @@ -57,32 +61,33 @@ class Parameterized( } private val coordinates: mutable.LinkedHashMap[Coordinate, Seq[(String, String)]] = { - validateDims(configDimensions) val dimCount = configDimensions.size val coordinateMap = mutable.LinkedHashMap[Coordinate, Seq[(String, String)]]() + val nextId: AtomicInteger = new AtomicInteger(1); def fillCoordinates( dimOffset: Int, - intermediateCoordinates: Map[String, String], + intermediateCoordinate: Map[String, String], intermediateConf: Seq[(String, String)]): Unit = { if (dimOffset == dimCount) { // we got one coordinate excludedCombinations.foreach { ec: Set[DimKv] => if (ec.forall { kv => - intermediateCoordinates.contains(kv.k) && intermediateCoordinates(kv.k) == kv.v + intermediateCoordinate.contains(kv.k) && intermediateCoordinate(kv.k) == kv.v }) { - println(s"Coordinate ${Coordinate(intermediateCoordinates)} excluded by $ec.") + println(s"Coordinate ${intermediateCoordinate} excluded by $ec.") return } } - coordinateMap(Coordinate(intermediateCoordinates)) = intermediateConf + coordinateMap(Coordinate(nextId.getAndIncrement(), intermediateCoordinate)) = + intermediateConf return } val dim = configDimensions(dimOffset) dim.dimValues.foreach { dimValue => fillCoordinates( dimOffset + 1, - intermediateCoordinates + (dim.name -> dimValue.name), + intermediateCoordinate + (dim.name -> dimValue.name), intermediateConf ++ dimValue.conf) } } @@ -95,7 +100,6 @@ class Parameterized( override def execute(suite: Suite): Boolean = { val runner: QueryRunner = new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) - val allQueries = suite.allQueryIds() val sessionSwitcher = suite.sessionSwitcher val testConf = suite.getTestConf() @@ -116,36 +120,40 @@ class Parameterized( val runQueryIds = queries.select(suite) - // warm up - (0 until warmupIterations).foreach { _ => - runQueryIds.foreach { queryId => - Parameterized.warmUp(suite.tableCreator(), queryId, suite.desc(), sessionSwitcher, runner) - } - } - - val results = coordinates.flatMap { entry => - val coordinate = entry._1 - val coordinateResults = (0 until iterations).flatMap { iteration => - println(s"Running tests (iteration $iteration) with coordinate $coordinate...") - runQueryIds.map { queryId => - Parameterized.runQuery( - runner, - suite.tableCreator(), - sessionSwitcher, + val results = (0 until iterations).flatMap { iteration => + runQueryIds.map { queryId => + val queryResult = + TestResultLine( queryId, - coordinate, - suite.desc(), - explain, - metrics) - } - }.toList - coordinateResults + coordinates.map { entry => + val coordinate = entry._1 + println(s"Running tests (iteration $iteration) with coordinate $coordinate...") + // warm up + (0 until warmupIterations).foreach { _ => + Parameterized.warmUp( + runner, + suite.tableCreator(), + sessionSwitcher, + queryId, + suite.desc()) + } + // run + Parameterized.runQuery( + runner, + suite.tableCreator(), + sessionSwitcher, + queryId, + coordinate, + suite.desc(), + explain, + metrics) + }.toList) + queryResult + } } - val dimNames = configDimensions.map(dim => dim.name) - - val passedCount = results.count(l => l.succeed) - val count = results.count(_ => true) + val succeededCount = results.count(l => l.succeeded()) + val totalCount = results.count(_ => true) // RAM stats println("Performing GC to collect RAM statistics... ") @@ -160,22 +168,37 @@ class Parameterized( println("") println("Test report: ") println("") - printf("Summary: %d out of %d queries passed. \n", passedCount, count) + printf( + "Summary: %d out of %d queries successfully run on all config combinations. \n", + succeededCount, + totalCount) println("") - TestResultLines(dimNames, metrics, results.filter(_.succeed)).print() + println("Configurations:") + coordinates.foreach { coord => + println(s"${coord._1.id}. ${coord._1}") + } + println("") + val succeeded = results.filter(_.succeeded()) + TestResultLines( + coordinates.size, + configDimensions, + metrics, + succeeded ++ TestResultLine.aggregate("all", succeeded)) + .print() println("") - if (passedCount == count) { + if (succeededCount == totalCount) { println("No failed queries. ") println("") } else { println("Failed queries: ") println("") - TestResultLines(dimNames, metrics, results.filter(!_.succeed)).print() + TestResultLines(coordinates.size, configDimensions, metrics, results.filter(!_.succeeded())) + .print() println("") } - if (passedCount != count) { + if (succeededCount != totalCount) { return false } true @@ -185,56 +208,84 @@ class Parameterized( case class DimKv(k: String, v: String) case class Dim(name: String, dimValues: Seq[DimValue]) case class DimValue(name: String, conf: Seq[(String, String)]) -case class Coordinate(coordinate: Map[String, String]) // [dim, dim value] - -case class TestResultLine( - queryId: String, - succeed: Boolean, - coordinate: Coordinate, - rowCount: Option[Long], - planningTimeMillis: Option[Long], - executionTimeMillis: Option[Long], - metrics: Map[String, Long], - errorMessage: Option[String]) +// coordinate: [dim, dim value] +case class Coordinate(id: Int, coordinate: Map[String, String]) { + override def toString: String = coordinate.mkString(", ") +} + +case class TestResultLine(queryId: String, coordinates: Seq[TestResultLine.Coord]) { + def succeeded(): Boolean = { + coordinates.forall(_.succeeded) + } +} object TestResultLine { - class Parser(dimNames: Seq[String], metricNames: Seq[String]) - extends TableRender.RowParser[TestResultLine] { + case class Coord( + coordinate: Coordinate, + succeeded: Boolean, + rowCount: Option[Long], + planningTimeMillis: Option[Long], + executionTimeMillis: Option[Long], + metrics: Map[String, Long], + errorMessage: Option[String]) + + class Parser(metricNames: Seq[String]) extends TableRender.RowParser[TestResultLine] { override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { val inc = rowAppender.incremental() inc.next().write(line.queryId) - inc.next().write(line.succeed) - dimNames.foreach { dimName => - val coordinate = line.coordinate.coordinate - if (!coordinate.contains(dimName)) { - throw new IllegalStateException("Dimension name not found" + dimName) - } - inc.next().write(coordinate(dimName)) - } - metricNames.foreach { metricName => - val metrics = line.metrics - inc.next().write(metrics.getOrElse(metricName, "N/A")) - } - inc.next().write(line.rowCount.getOrElse("N/A")) - inc.next().write(line.planningTimeMillis.getOrElse("N/A")) - inc.next().write(line.executionTimeMillis.getOrElse("N/A")) + val coords = line.coordinates + coords.foreach(coord => inc.next().write(coord.succeeded)) + coords.foreach(coord => inc.next().write(coord.rowCount)) + metricNames.foreach(metricName => + coords.foreach(coord => inc.next().write(coord.metrics(metricName)))) + coords.foreach(coord => inc.next().write(coord.planningTimeMillis)) + coords.foreach(coord => inc.next().write(coord.executionTimeMillis)) + } + } + + def aggregate(name: String, lines: Iterable[TestResultLine]): Iterable[TestResultLine] = { + if (lines.isEmpty) { + return Nil + } + + if (lines.size == 1) { + return Nil } + + List(lines.reduce { (left, right) => + TestResultLine(name, left.coordinates.zip(right.coordinates).map { + case (leftCoord, rightCoord) => + assert(leftCoord.coordinate == rightCoord.coordinate) + Coord( + leftCoord.coordinate, + leftCoord.succeeded && rightCoord.succeeded, + (leftCoord.rowCount, rightCoord.rowCount).onBothProvided(_ + _), + (leftCoord.planningTimeMillis, rightCoord.planningTimeMillis).onBothProvided(_ + _), + (leftCoord.executionTimeMillis, rightCoord.executionTimeMillis).onBothProvided(_ + _), + (leftCoord.metrics, rightCoord.metrics).sumUp, + (leftCoord.errorMessage ++ rightCoord.errorMessage).reduceOption(_ + ", " + _)) + }) + }) } } case class TestResultLines( - dimNames: Seq[String], + coordCount: Int, + configDimensions: Seq[Dim], metricNames: Seq[String], lines: Iterable[TestResultLine]) { def print(): Unit = { - val fields = ListBuffer[String]("Query ID", "Succeeded") - dimNames.foreach(dimName => fields.append(dimName)) - metricNames.foreach(metricName => fields.append(metricName)) - fields.append("Row Count") - fields.append("Planning Time (Millis)") - fields.append("Query Time (Millis)") - val render = TableRender.plain[TestResultLine](fields: _*)( - new TestResultLine.Parser(dimNames, metricNames)) + val fields = ListBuffer[Field](Field.Leaf("Query ID")) + val coordFields = (1 to coordCount).map(id => Field.Leaf(id.toString)) + + fields.append(Field.Branch("Succeeded", coordFields)) + fields.append(Field.Branch("Row Count", coordFields)) + metricNames.foreach(metricName => fields.append(Field.Branch(metricName, coordFields))) + fields.append(Field.Branch("Planning Time (Millis)", coordFields)) + fields.append(Field.Branch("Query Time (Millis)", coordFields)) + + val render = + TableRender.create[TestResultLine](fields: _*)(new TestResultLine.Parser(metricNames)) lines.foreach { line => render.appendRow(line) @@ -253,10 +304,10 @@ object Parameterized { coordinate: Coordinate, desc: String, explain: Boolean, - metrics: Array[String]) = { + metrics: Array[String]): TestResultLine.Coord = { println(s"Running query: $id...") try { - val testDesc = "Gluten Spark %s %s %s".format(desc, id, coordinate) + val testDesc = "Gluten Spark %s [%s] %s".format(desc, id, coordinate) sessionSwitcher.useSession(coordinate.toString, testDesc) runner.createTables(creator, sessionSwitcher.spark()) val result = @@ -265,10 +316,9 @@ object Parameterized { println( s"Successfully ran query $id. " + s"Returned row count: ${resultRows.length}") - TestResultLine( - id, - succeed = true, + TestResultLine.Coord( coordinate, + succeeded = true, Some(resultRows.length), Some(result.planningTimeMillis), Some(result.executionTimeMillis), @@ -280,16 +330,16 @@ object Parameterized { println( s"Error running query $id. " + s" Error: ${error.get}") - TestResultLine(id, succeed = false, coordinate, None, None, None, Map.empty, error) + TestResultLine.Coord(coordinate, succeeded = false, None, None, None, Map.empty, error) } } - private[integration] def warmUp( + private def warmUp( + runner: QueryRunner, creator: TableCreator, - id: String, - desc: String, sessionSwitcher: SparkSessionSwitcher, - runner: QueryRunner): Unit = { + id: String, + desc: String): Unit = { println(s"Warming up: Running query: $id...") try { val testDesc = "Gluten Spark %s %s warm up".format(desc, id) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala index 540abbf454c3..de09d925e4d2 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala @@ -114,9 +114,9 @@ object Queries { val inc = rowAppender.incremental() inc.next().write(line.queryId) inc.next().write(line.testPassed) - inc.next().write(line.rowCount.getOrElse("N/A")) - inc.next().write(line.planningTimeMillis.getOrElse("N/A")) - inc.next().write(line.executionTimeMillis.getOrElse("N/A")) + inc.next().write(line.rowCount) + inc.next().write(line.planningTimeMillis) + inc.next().write(line.executionTimeMillis) } } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala index 596c293e473e..d7b6ffff893c 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala @@ -81,10 +81,10 @@ case class QueriesCompare( println("") } - var all = QueriesCompare.aggregate(results, "all") + var all = QueriesCompare.aggregate("all", results) if (passedCount != count) { - all = QueriesCompare.aggregate(succeed, "succeeded") ::: all + all = QueriesCompare.aggregate("succeeded", succeed) ::: all } println("Overall: ") @@ -123,13 +123,13 @@ object QueriesCompare { } else None inc.next().write(line.queryId) inc.next().write(line.testPassed) - inc.next().write(line.expectedRowCount.getOrElse("N/A")) - inc.next().write(line.actualRowCount.getOrElse("N/A")) - inc.next().write(line.expectedPlanningTimeMillis.getOrElse("N/A")) - inc.next().write(line.actualPlanningTimeMillis.getOrElse("N/A")) - inc.next().write(line.expectedExecutionTimeMillis.getOrElse("N/A")) - inc.next().write(line.actualExecutionTimeMillis.getOrElse("N/A")) - inc.next().write(speedUp.map("%.2f%%".format(_)).getOrElse("N/A")) + inc.next().write(line.expectedRowCount) + inc.next().write(line.actualRowCount) + inc.next().write(line.expectedPlanningTimeMillis) + inc.next().write(line.actualPlanningTimeMillis) + inc.next().write(line.expectedExecutionTimeMillis) + inc.next().write(line.actualExecutionTimeMillis) + inc.next().write(speedUp.map("%.2f%%".format(_))) } } } @@ -152,7 +152,7 @@ object QueriesCompare { render.print(System.out) } - private def aggregate(succeed: List[TestResultLine], name: String): List[TestResultLine] = { + private def aggregate(name: String, succeed: List[TestResultLine]): List[TestResultLine] = { if (succeed.isEmpty) { return Nil } @@ -160,25 +160,13 @@ object QueriesCompare { succeed.reduce((r1, r2) => TestResultLine( name, - testPassed = true, - if (r1.expectedRowCount.nonEmpty && r2.expectedRowCount.nonEmpty) - Some(r1.expectedRowCount.get + r2.expectedRowCount.get) - else None, - if (r1.actualRowCount.nonEmpty && r2.actualRowCount.nonEmpty) - Some(r1.actualRowCount.get + r2.actualRowCount.get) - else None, - if (r1.expectedPlanningTimeMillis.nonEmpty && r2.expectedPlanningTimeMillis.nonEmpty) - Some(r1.expectedPlanningTimeMillis.get + r2.expectedPlanningTimeMillis.get) - else None, - if (r1.actualPlanningTimeMillis.nonEmpty && r2.actualPlanningTimeMillis.nonEmpty) - Some(r1.actualPlanningTimeMillis.get + r2.actualPlanningTimeMillis.get) - else None, - if (r1.expectedExecutionTimeMillis.nonEmpty && r2.expectedExecutionTimeMillis.nonEmpty) - Some(r1.expectedExecutionTimeMillis.get + r2.expectedExecutionTimeMillis.get) - else None, - if (r1.actualExecutionTimeMillis.nonEmpty && r2.actualExecutionTimeMillis.nonEmpty) - Some(r1.actualExecutionTimeMillis.get + r2.actualExecutionTimeMillis.get) - else None, + r1.testPassed && r2.testPassed, + (r1.expectedRowCount, r2.expectedRowCount).onBothProvided(_ + _), + (r1.actualRowCount, r2.actualRowCount).onBothProvided(_ + _), + (r1.expectedPlanningTimeMillis, r2.expectedPlanningTimeMillis).onBothProvided(_ + _), + (r1.actualPlanningTimeMillis, r2.actualPlanningTimeMillis).onBothProvided(_ + _), + (r1.expectedExecutionTimeMillis, r2.expectedExecutionTimeMillis).onBothProvided(_ + _), + (r1.actualExecutionTimeMillis, r2.actualExecutionTimeMillis).onBothProvided(_ + _), None))) } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala index 4cded2848b6e..2b1cca61e3f4 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/TableRender.scala @@ -20,7 +20,7 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.StringUtils import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender -import java.io.{OutputStream, PrintStream} +import java.io.{ByteArrayOutputStream, OutputStream, PrintStream} import scala.collection.mutable trait TableRender[ROW <: Any] { @@ -31,7 +31,8 @@ trait TableRender[ROW <: Any] { object TableRender { def create[ROW <: Any](fields: Field*)(implicit parser: RowParser[ROW]): TableRender[ROW] = { assert(fields.nonEmpty) - new Impl[ROW](Schema(fields), parser) + // Deep copy to avoid duplications (In case caller reuses a sub-tree). + new Impl[ROW](Schema(fields.map(_.makeCopy())), parser) } def plain[ROW <: Any](fields: String*)(implicit parser: RowParser[ROW]): TableRender[ROW] = { @@ -40,8 +41,10 @@ object TableRender { } trait Field { + def id(): Int = System.identityHashCode(this) def name: String def leafs: Seq[Field.Leaf] + def makeCopy(): Field } object Field { @@ -57,9 +60,12 @@ object TableRender { children.map(child => leafsOf(child)).reduce(_ ++ _) } } + + override def makeCopy(): Field = copy(name, children.map(_.makeCopy())) } case class Leaf(override val name: String) extends Field { override val leafs: Seq[Leaf] = List(this) + override def makeCopy(): Field = copy() } } @@ -109,7 +115,7 @@ object TableRender { schema.leafs.zipWithIndex.foreach { case (leaf, i) => val dataWidth = dataWidths(i) - widthMap += (System.identityHashCode(leaf) -> (dataWidth max (leaf.name.length + 2))) + widthMap += (leaf.id() -> (dataWidth max (leaf.name.length + 2))) } schema.fields.foreach { root => @@ -122,12 +128,12 @@ object TableRender { .toInt children.foreach(child => updateWidth(child, leafLowerBound * child.leafs.size)) val childrenWidth = - children.map(child => widthMap(System.identityHashCode(child))).sum + children.map(child => widthMap(child.id())).sum val width = childrenWidth + children.size - 1 - val hash = System.identityHashCode(branch) + val hash = branch.id() widthMap += hash -> width case leaf @ Field.Leaf(name) => - val hash = System.identityHashCode(leaf) + val hash = leaf.id() val newWidth = widthMap(hash) max lowerBound widthMap.put(hash, newWidth) case _ => new IllegalStateException() @@ -146,9 +152,9 @@ object TableRender { val schemaLine = cells .map { case Given(field) => - (field.name, widthMap(System.identityHashCode(field))) + (field.name, widthMap(field.id())) case PlaceHolder(leaf) => - ("", widthMap(System.identityHashCode(leaf))) + ("", widthMap(leaf.id())) } .map { case (name, width) => @@ -168,7 +174,7 @@ object TableRender { val separationLine = schema.leafs .map { leaf => - widthMap(System.identityHashCode(leaf)) + widthMap(leaf.id()) } .map { width => new String(Array.tabulate(width)(_ => '-')) @@ -182,7 +188,7 @@ object TableRender { .zip(schema.leafs) .map { case (value, leaf) => - (value, widthMap(System.identityHashCode(leaf))) + (value, widthMap(leaf.id())) } .map { case (value, width) => @@ -194,6 +200,12 @@ object TableRender { printer.flush() } + + override def toString: String = { + val out = new ByteArrayOutputStream() + print(out) + out.toString + } } trait RowParser[ROW <: Any] { @@ -302,7 +314,13 @@ object TableRender { override def write(value: Any): Unit = { assert(field.isInstanceOf[Field.Leaf]) - mutableRow(column) = value.toString + mutableRow(column) = toString(value) + } + + private def toString(value: Any): String = value match { + case Some(v) => toString(v) + case None => "N/A" + case other => other.toString } } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala new file mode 100644 index 000000000000..6046ae4aaa35 --- /dev/null +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gluten.integration + +package object action { + implicit class DualOptionsOps[T](value: (Option[T], Option[T])) { + def onBothProvided[R](func: (T, T) => R): Option[R] = { + if (value._1.isEmpty || value._2.isEmpty) { + return None + } + Some(func(value._1.get, value._2.get)) + } + } + + implicit class DualMetricsOps(value: (Map[String, Long], Map[String, Long])) { + def sumUp: Map[String, Long] = { + assert(value._1.keySet == value._2.keySet) + value._1.map { case (k, v) => k -> (v + value._2(k)) } + } + } +} diff --git a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala index ce7b0974ce8b..1efc72148928 100644 --- a/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala +++ b/tools/gluten-it/common/src/test/java/org/apache/gluten/integration/action/TableRenderTest.scala @@ -99,11 +99,32 @@ object TableRenderTest { Console.out.println() } + def case5(): Unit = { + val leafs = List(Leaf("1"), Leaf("2"), Leaf("3"), Leaf("4")) + val render: TableRender[Seq[String]] = TableRender.create( + Leaf("Query ID"), + Branch("Succeeded", leafs), + Branch("Row Count", leafs))(new RowParser[Seq[String]] { + override def parse(rowFactory: FieldAppender.RowAppender, row: Seq[String]): Unit = { + val inc = rowFactory.incremental() + row.foreach(ceil => inc.next().write(ceil)) + } + }) + + render.appendRow( + List("q1", "true", "true", "true && true && true && true", "true", "1", "1", "1", "1")) + render.appendRow( + List("q2", "true", "true", "true", "true", "100000", "100000", "100000", "100000")) + render.print(Console.out) + Console.out.println() + } + def main(args: Array[String]): Unit = { case0() case1() case2() case3() case4() + case5() } } From c00ffe7db44dbcd6f1f0a41067307adeeaccad47 Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Wed, 12 Jun 2024 14:48:32 +0530 Subject: [PATCH 255/402] [VL] Update supportColumnarShuffleExec for Velox to consider enableColumnarShuffle config (#6055) --- .../org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 060a0c4cbd34..6bc7df98cca2 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -351,9 +351,9 @@ object VeloxBackendSettings extends BackendSettingsApi { } override def supportColumnarShuffleExec(): Boolean = { - GlutenConfig.getConf.isUseColumnarShuffleManager || - GlutenConfig.getConf.isUseCelebornShuffleManager || - GlutenConfig.getConf.isUseUniffleShuffleManager + GlutenConfig.getConf.enableColumnarShuffle && (GlutenConfig.getConf.isUseColumnarShuffleManager + || GlutenConfig.getConf.isUseCelebornShuffleManager + || GlutenConfig.getConf.isUseUniffleShuffleManager) } override def enableJoinKeysRewrite(): Boolean = false From 7445f02c9414c98956d1ad4d9318f6e545e1740c Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Wed, 12 Jun 2024 10:48:02 -0500 Subject: [PATCH 256/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240612) (#6050) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240612) * fix ut due to https://github.com/ClickHouse/ClickHouse/pull/64427 Don't know the reason, but this PR changed the 'input_format_parquetmax_block_size' to 'DEFAULT_block_size', which is 65409, causing the MergeTree related tests to fail --------- Co-authored-by: kyligence-git Co-authored-by: liuneng1994 --- ...utenClickHouseMergeTreeOptimizeSuite.scala | 60 +++++++++---------- ...ickHouseMergeTreePathBasedWriteSuite.scala | 30 +++++----- .../GlutenClickHouseMergeTreeWriteSuite.scala | 28 ++++----- .../GlutenClickHouseTableAfterRestart.scala | 2 +- cpp-ch/clickhouse.version | 4 +- 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index 650bbcc7b32c..6f8b4d93beb6 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -83,7 +83,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) assert( - countFiles(new File(s"$basePath/lineitem_mergetree_optimize")) == 462 + countFiles(new File(s"$basePath/lineitem_mergetree_optimize")) == 641 ) // many merged parts } } @@ -162,12 +162,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() assert(ret.apply(0).get(0) == 600572) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 372) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 348) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 239) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 236) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 241) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 238) } spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") // the second VACUUM will remove some empty folders @@ -199,18 +199,18 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() assert(ret.apply(0).get(0) == 600572) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 516) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 448) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 306) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 294) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 308) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 296) } spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 276) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 272) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 282) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 278) } val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() @@ -236,18 +236,18 @@ class GlutenClickHouseMergeTreeOptimizeSuite val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() assert(ret.apply(0).get(0) == 600572) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 516) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 448) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 306) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 294) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 308) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 296) } spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 276) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 272) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 282) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 278) } val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() @@ -256,7 +256,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite } test("test mergetree optimize with optimize.minFileSize and optimize.maxFileSize") { - withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "838000") { + withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "83800000") { // 3 from 37 parts are larger than this, so after optimize there should be 4 parts: // 3 original parts and 1 merged part spark.sql(s""" @@ -275,11 +275,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 99) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 75) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. // this case will create a checkpoint - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 105) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 81) } val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() @@ -299,10 +299,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 75) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 104) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 85) } val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() @@ -315,10 +315,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 77) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 75) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 93) + assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 90) } val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() @@ -346,7 +346,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite assert(ret.apply(0).get(0) == 600572) assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 499 else 528 + if (sparkVersion.equals("3.2")) 475 else 501 }) spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") @@ -360,7 +360,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite test("test skip index after optimize") { withSQLConf( - "spark.databricks.delta.optimize.maxFileSize" -> "2000000", + "spark.databricks.delta.optimize.maxFileSize" -> "100000000", "spark.sql.adaptive.enabled" -> "false") { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_index; @@ -403,7 +403,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite test("test mergetree optimize with the path based table") { val dataPath = s"$basePath/lineitem_mergetree_optimize_path_based" clearDataPath(dataPath) - withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "838000") { + withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "83800000") { // 3 from 37 parts are larger than this, so after optimize there should be 4 parts: // 3 original parts and 1 merged part @@ -422,9 +422,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(dataPath)) == 99) + assert(countFiles(new File(dataPath)) == 75) } else { - assert(countFiles(new File(dataPath)) == 105) + assert(countFiles(new File(dataPath)) == 81) } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() @@ -445,9 +445,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(dataPath)) == 93) + assert(countFiles(new File(dataPath)) == 75) } else { - assert(countFiles(new File(dataPath)) == 104) + assert(countFiles(new File(dataPath)) == 85) } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() @@ -461,9 +461,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(dataPath)) == 77) + assert(countFiles(new File(dataPath)) == 75) } else { - assert(countFiles(new File(dataPath)) == 93) + assert(countFiles(new File(dataPath)) == 90) } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala index 93f22baa2575..c8c6307aba06 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala @@ -154,7 +154,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -266,7 +266,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .equals("l_returnflag,l_linestatus")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -286,7 +286,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .load(dataPath) .where("l_shipdate = date'1998-09-02'") .collect() - assert(result.apply(0).get(0) == 110501) + assert(result.length == 183) } test("test mergetree path based insert overwrite partitioned table with small table, static") { @@ -406,7 +406,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(addFiles.size == 6) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) assert(filePaths.size == 2) - assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) } val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) @@ -436,8 +436,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite // 2 parts belong to the second batch (1 actual updated part, 1 passively updated). assert(addFiles.size == 6) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) - assert(filePaths.size == 2) - assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assert(filePaths.size == 3) + assert(Array(1, 2, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) } val df = spark.read @@ -477,7 +477,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(addFiles.size == 6) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) assert(filePaths.size == 2) - assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.delete("mod(l_orderkey, 3) = 2") @@ -635,7 +635,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert(addFiles.map(_.rows).sum == 600572) } @@ -941,7 +941,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -1126,14 +1126,14 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert(addFiles.map(_.rows).sum == 600572) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } assert(plans.size == 1) - assert(plans(0).metrics("selectedMarksPk").value === 17) + assert(plans(0).metrics("selectedMarksPk").value === 15) assert(plans(0).metrics("totalMarksPk").value === 74) } } @@ -1201,7 +1201,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val mergetreeScan = scanExec(0) assert(ret.apply(0).get(0) == 2) val marks = mergetreeScan.metrics("selectedMarks").value - assert(marks == 1) + assert(marks == 2) val directory = new File(dataPath) // find a folder whose name is like 48b70783-b3b8-4bf8-9c52-5261aead8e3e_0_006 @@ -1285,7 +1285,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -1320,7 +1320,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val fileFilter = new WildcardFileFilter("*_0_*") var dataFileList = dataPathFile.list(fileFilter) - assert(dataFileList.size == 6) + assert(dataFileList.size == 5) // re-create the same table val dataPath2 = s"$basePath/lineitem_mergetree_5219_s" @@ -1339,7 +1339,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(dataPathFile.isDirectory && dataPathFile.isDirectory) dataFileList = dataPathFile.list(fileFilter) - assert(dataFileList.size == 6) + assert(dataFileList.size == 5) } } // scalastyle:off line.size.limit diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala index 439a1b58fd4f..679cea37ba67 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala @@ -140,7 +140,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -385,7 +385,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(addFiles.size == 6) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) assert(filePaths.size == 2) - assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) } val sql2 = @@ -458,7 +458,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(addFiles.size == 6) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) assert(filePaths.size == 2) - assert(Array(2, 4).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) } { @@ -667,7 +667,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert(addFiles.map(_.rows).sum == 600572) } } @@ -1155,7 +1155,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -1384,14 +1384,14 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert(addFiles.map(_.rows).sum == 600572) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } assert(plans.size == 1) - assert(plans(0).metrics("selectedMarksPk").value === 17) + assert(plans(0).metrics("selectedMarksPk").value === 15) assert(plans(0).metrics("totalMarksPk").value === 74) } } @@ -1470,14 +1470,14 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert(addFiles.map(_.rows).sum == 600572) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } assert(plans.size == 1) - assert(plans(0).metrics("selectedMarksPk").value === 17) + assert(plans(0).metrics("selectedMarksPk").value === 15) assert(plans(0).metrics("totalMarksPk").value === 74) } } @@ -1630,7 +1630,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(result.size == 1) assert(result(0).getDouble(0).toString.substring(0, 6).equals("5.3379")) - checkSelectedMarksCnt(df, 29) + checkSelectedMarksCnt(df, 24) }) } @@ -1674,7 +1674,7 @@ class GlutenClickHouseMergeTreeWriteSuite val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) + assert(addFiles.size == 5) assert( addFiles.map(_.rows).sum == 600572) @@ -1715,7 +1715,7 @@ class GlutenClickHouseMergeTreeWriteSuite val fileFilter = new WildcardFileFilter("*_0_*") var dataFileList = dataPath.list(fileFilter) - assert(dataFileList.size == 6) + assert(dataFileList.size == 5) // test with the normal table spark.sql(s""" @@ -1796,7 +1796,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(dataPath.isDirectory && dataPath.isDirectory) dataFileList = dataPath.list(fileFilter) - assert(dataFileList.size == 6) + assert(dataFileList.size == 5) // re-create the same table for (i <- 0 until 10) { @@ -1818,7 +1818,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(dataPath.isDirectory && dataPath.isDirectory) dataFileList = dataPath.list(fileFilter) - assert(dataFileList.size == 6) + assert(dataFileList.size == 5) } test("test mergetree with primary keys filter pruning by driver") { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala index baf79436cf8b..ab11c1e0c201 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala @@ -196,7 +196,7 @@ class GlutenClickHouseTableAfterRestart val stats1 = ClickhouseSnapshot.deltaScanCache.stats() assert(stats1.missCount() - oldMissingCount1 == 1) val stats2 = ClickhouseSnapshot.addFileToAddMTPCache.stats() - assert(stats2.missCount() - oldMissingCount2 == 6) + assert(stats2.missCount() - oldMissingCount2 == 5) } diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index a5ca8d8dd0e1..991edb4f20e5 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240608 -CH_COMMIT=b5050282335 +CH_BRANCH=rebase_ch/20240612 +CH_COMMIT=e13cab114c5 From 2ae80afbca24e92fca5c9c5d0849a37a5b5c15fd Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Thu, 13 Jun 2024 06:27:10 +0530 Subject: [PATCH 257/402] [CORE] Use SortShuffleManager instance in ColumnarShuffleManager (#6022) --- .../gluten/execution/FallbackSuite.scala | 26 +++- .../shuffle/sort/ColumnarShuffleManager.scala | 121 ++++++------------ 2 files changed, 67 insertions(+), 80 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index 15a71ceb587b..27d191b9ee05 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -20,8 +20,9 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.extension.GlutenPlan import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.{ColumnarShuffleExchangeExec, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEShuffleReadExec} +import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPlanHelper { protected val rootPath: String = getClass.getResource("/").getPath @@ -71,6 +72,29 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl collect(plan) { case v: VeloxColumnarToRowExec => v }.size } + private def collectColumnarShuffleExchange(plan: SparkPlan): Int = { + collect(plan) { case c: ColumnarShuffleExchangeExec => c }.size + } + + private def collectShuffleExchange(plan: SparkPlan): Int = { + collect(plan) { case c: ShuffleExchangeExec => c }.size + } + + test("fallback with shuffle manager") { + withSQLConf(GlutenConfig.COLUMNAR_SHUFFLE_ENABLED.key -> "false") { + runQueryAndCompare("select c1, count(*) from tmp1 group by c1") { + df => + val plan = df.queryExecution.executedPlan + + assert(collectColumnarShuffleExchange(plan) == 0) + assert(collectShuffleExchange(plan) == 1) + + val wholeQueryColumnarToRow = collectColumnarToRow(plan) + assert(wholeQueryColumnarToRow == 2) + } + } + } + test("fallback with collect") { withSQLConf(GlutenConfig.COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD.key -> "1") { runQueryAndCompare("SELECT count(*) FROM tmp1") { diff --git a/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala b/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala index d8ba78cb98fd..06c6e6c0ea5a 100644 --- a/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala +++ b/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala @@ -20,7 +20,6 @@ import org.apache.spark.{ShuffleDependency, SparkConf, SparkEnv, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.serializer.SerializerManager import org.apache.spark.shuffle._ -import org.apache.spark.shuffle.api.ShuffleExecutorComponents import org.apache.spark.shuffle.sort.SortShuffleManager.canUseBatchFetch import org.apache.spark.storage.BlockId import org.apache.spark.util.collection.OpenHashSet @@ -28,13 +27,12 @@ import org.apache.spark.util.collection.OpenHashSet import java.io.InputStream import java.util.concurrent.ConcurrentHashMap -import scala.collection.JavaConverters._ - class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { import ColumnarShuffleManager._ - private lazy val shuffleExecutorComponents = loadShuffleExecutorComponents(conf) + private[this] lazy val sortShuffleManager: SortShuffleManager = new SortShuffleManager(conf) + override val shuffleBlockResolver = new IndexShuffleBlockResolver(conf) /** A mapping from shuffle ids to the number of mappers producing output for those shuffles. */ @@ -49,23 +47,9 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin new ColumnarShuffleHandle[K, V]( shuffleId, dependency.asInstanceOf[ColumnarShuffleDependency[K, V, V]]) - } else if (SortShuffleWriter.shouldBypassMergeSort(conf, dependency)) { - // If there are fewer than spark.shuffle.sort.bypassMergeThreshold partitions and we don't - // need map-side aggregation, then write numPartitions files directly and just concatenate - // them at the end. This avoids doing serialization and deserialization twice to merge - // together the spilled files, which would happen with the normal code path. The downside is - // having multiple files open at a time and thus more memory allocated to buffers. - new BypassMergeSortShuffleHandle[K, V]( - shuffleId, - dependency.asInstanceOf[ShuffleDependency[K, V, V]]) - } else if (SortShuffleManager.canUseSerializedShuffle(dependency)) { - // Otherwise, try to buffer map outputs in a serialized form, since this is more efficient: - new SerializedShuffleHandle[K, V]( - shuffleId, - dependency.asInstanceOf[ShuffleDependency[K, V, V]]) } else { - // Otherwise, buffer map outputs in a deserialized form: - new BaseShuffleHandle(shuffleId, dependency) + // Otherwise call default SortShuffleManager + sortShuffleManager.registerShuffle(shuffleId, dependency) } } @@ -75,39 +59,19 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin mapId: Long, context: TaskContext, metrics: ShuffleWriteMetricsReporter): ShuffleWriter[K, V] = { - val mapTaskIds = - taskIdMapsForShuffle.computeIfAbsent(handle.shuffleId, _ => new OpenHashSet[Long](16)) - mapTaskIds.synchronized { - mapTaskIds.add(context.taskAttemptId()) - } - val env = SparkEnv.get handle match { case columnarShuffleHandle: ColumnarShuffleHandle[K @unchecked, V @unchecked] => + val mapTaskIds = + taskIdMapsForShuffle.computeIfAbsent(handle.shuffleId, _ => new OpenHashSet[Long](16)) + mapTaskIds.synchronized { + mapTaskIds.add(context.taskAttemptId()) + } GlutenShuffleWriterWrapper.genColumnarShuffleWriter( shuffleBlockResolver, columnarShuffleHandle, mapId, metrics) - case unsafeShuffleHandle: SerializedShuffleHandle[K @unchecked, V @unchecked] => - new UnsafeShuffleWriter( - env.blockManager, - context.taskMemoryManager(), - unsafeShuffleHandle, - mapId, - context, - env.conf, - metrics, - shuffleExecutorComponents) - case bypassMergeSortHandle: BypassMergeSortShuffleHandle[K @unchecked, V @unchecked] => - new BypassMergeSortShuffleWriter( - env.blockManager, - bypassMergeSortHandle, - mapId, - env.conf, - metrics, - shuffleExecutorComponents) - case other: BaseShuffleHandle[K @unchecked, V @unchecked, _] => - new SortShuffleWriter(other, mapId, context, shuffleExecutorComponents) + case _ => sortShuffleManager.getWriter(handle, mapId, context, metrics) } } @@ -123,17 +87,17 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin endPartition: Int, context: TaskContext, metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = { - val (blocksByAddress, canEnableBatchFetch) = { - GlutenShuffleUtils.getReaderParam( - handle, - startMapIndex, - endMapIndex, - startPartition, - endPartition) - } - val shouldBatchFetch = - canEnableBatchFetch && canUseBatchFetch(startPartition, endPartition, context) if (handle.isInstanceOf[ColumnarShuffleHandle[_, _]]) { + val (blocksByAddress, canEnableBatchFetch) = { + GlutenShuffleUtils.getReaderParam( + handle, + startMapIndex, + endMapIndex, + startPartition, + endPartition) + } + val shouldBatchFetch = + canEnableBatchFetch && canUseBatchFetch(startPartition, endPartition, context) new BlockStoreShuffleReader( handle.asInstanceOf[BaseShuffleHandle[K, _, C]], blocksByAddress, @@ -143,44 +107,43 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin shouldBatchFetch = shouldBatchFetch ) } else { - new BlockStoreShuffleReader( - handle.asInstanceOf[BaseShuffleHandle[K, _, C]], - blocksByAddress, + sortShuffleManager.getReader( + handle, + startMapIndex, + endMapIndex, + startPartition, + endPartition, context, - metrics, - shouldBatchFetch = shouldBatchFetch - ) + metrics) } } /** Remove a shuffle's metadata from the ShuffleManager. */ override def unregisterShuffle(shuffleId: Int): Boolean = { - Option(taskIdMapsForShuffle.remove(shuffleId)).foreach { - mapTaskIds => - mapTaskIds.iterator.foreach { - mapId => shuffleBlockResolver.removeDataByMap(shuffleId, mapId) - } + if (taskIdMapsForShuffle.contains(shuffleId)) { + Option(taskIdMapsForShuffle.remove(shuffleId)).foreach { + mapTaskIds => + mapTaskIds.iterator.foreach { + mapId => shuffleBlockResolver.removeDataByMap(shuffleId, mapId) + } + } + true + } else { + sortShuffleManager.unregisterShuffle(shuffleId) } - true } /** Shut down this ShuffleManager. */ override def stop(): Unit = { - shuffleBlockResolver.stop() + if (!taskIdMapsForShuffle.isEmpty) { + shuffleBlockResolver.stop() + } else { + sortShuffleManager.stop + } } } object ColumnarShuffleManager extends Logging { - private def loadShuffleExecutorComponents(conf: SparkConf): ShuffleExecutorComponents = { - val executorComponents = ShuffleDataIOUtils.loadShuffleDataIO(conf).executor() - val extraConfigs = conf.getAllWithPrefix(ShuffleDataIOUtils.SHUFFLE_SPARK_CONF_PREFIX).toMap - executorComponents.initializeExecutor( - conf.getAppId, - SparkEnv.get.executorId, - extraConfigs.asJava) - executorComponents - } - private def bypassDecompressionSerializerManger = new SerializerManager( SparkEnv.get.serializer, From 468000c43efe7d54910431c7268768c4cb2d0410 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 13 Jun 2024 09:18:11 +0800 Subject: [PATCH 258/402] [CORE] Rephrase metric names using "totaltime" as prefix (#6058) --- .../backendsapi/clickhouse/CHMetricsApi.scala | 34 ++++----- .../execution/CHGenerateExecTransformer.scala | 2 +- .../backendsapi/velox/VeloxMetricsApi.scala | 76 ++++++++----------- .../execution/GenerateExecTransformer.scala | 2 +- 4 files changed, 51 insertions(+), 63 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala index 350548e981d6..a5fb4a1853e8 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHMetricsApi.scala @@ -144,7 +144,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genFilterTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = @@ -160,7 +160,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genProjectTransformerMetricsUpdater( @@ -181,7 +181,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "resizeOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of resize output rows"), "aggregatingTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of aggregating"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genHashAggregateTransformerMetricsUpdater( @@ -198,7 +198,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genExpandTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = @@ -213,17 +213,15 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { Map( "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"), "bytesSpilled" -> SQLMetrics.createSizeMetric(sparkContext, "shuffle bytes spilled"), - "computePidTime" -> SQLMetrics.createNanoTimingMetric( - sparkContext, - "totaltime to compute pid"), - "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to split"), - "IOTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to disk io"), + "computePidTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to compute pid"), + "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to split"), + "IOTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to disk io"), "serializeTime" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime to block serialization"), - "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to spill"), - "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to compress"), - "prepareTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to prepare"), + "time to block serialization"), + "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to spill"), + "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to compress"), + "prepareTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to prepare"), "avgReadBatchNumRows" -> SQLMetrics .createAverageMetric(sparkContext, "avg read batch num rows"), "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"), @@ -245,7 +243,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genWindowTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = @@ -262,7 +260,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { Map( "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"), "numOutputBatches" -> SQLMetrics.createMetric(sparkContext, "number of output batches"), - "convertTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time to convert") + "convertTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to convert") ) override def genLimitTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = @@ -275,7 +273,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genLimitTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = @@ -291,7 +289,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def genSortTransformerMetricsUpdater(metrics: Map[String, SQLMetric]): MetricsUpdater = @@ -339,7 +337,7 @@ class CHMetricsApi extends MetricsApi with Logging with LogLevelUtil { SQLMetrics.createTimingMetric(sparkContext, "time of postProjection"), "probeTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of probe"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time"), + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time"), "fillingRightJoinSideTime" -> SQLMetrics.createTimingMetric( sparkContext, "filling right join side time"), diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHGenerateExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHGenerateExecTransformer.scala index f1e7d305a358..733c0a472814 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHGenerateExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHGenerateExecTransformer.scala @@ -54,7 +54,7 @@ case class CHGenerateExecTransformer( "extraTime" -> SQLMetrics.createTimingMetric(sparkContext, "extra operators time"), "inputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for data"), "outputWaitTime" -> SQLMetrics.createTimingMetric(sparkContext, "time of waiting for output"), - "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "total time") + "totalTime" -> SQLMetrics.createTimingMetric(sparkContext, "time") ) override def metricsUpdater(): MetricsUpdater = new GenerateMetricsUpdater(metrics) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala index 0811d71d16b8..c2696de50641 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala @@ -41,7 +41,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { sparkContext: SparkContext): Map[String, SQLMetric] = { Map( "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of input iterator"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of input iterator"), "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors") ) @@ -62,7 +62,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of batch scan"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of batch scan"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "scanTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "scan time"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), @@ -93,10 +93,8 @@ class VeloxMetricsApi extends MetricsApi with Logging { "rawInputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of raw input bytes"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "scanTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of scan"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric( - sparkContext, - "totaltime of scan and filter"), + "scanTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of scan"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of scan and filter"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"), @@ -133,10 +131,8 @@ class VeloxMetricsApi extends MetricsApi with Logging { "rawInputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of raw input bytes"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "scanTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of scan"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric( - sparkContext, - "totaltime of scan and filter"), + "scanTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of scan"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of scan and filter"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"), @@ -171,7 +167,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of filter"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of filter"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -187,7 +183,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of project"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of project"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -205,7 +201,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "aggOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "aggOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), "aggCpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), - "aggWallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of aggregation"), + "aggWallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of aggregation"), "aggPeakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "aggNumMemoryAllocations" -> SQLMetrics.createMetric( sparkContext, @@ -222,13 +218,13 @@ class VeloxMetricsApi extends MetricsApi with Logging { "rowConstruction cpu wall time count"), "rowConstructionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of rowConstruction"), + "time of rowConstruction"), "extractionCpuCount" -> SQLMetrics.createMetric( sparkContext, "extraction cpu wall time count"), "extractionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of extraction"), + "time of extraction"), "finalOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of final output rows"), "finalOutputVectors" -> SQLMetrics.createMetric( sparkContext, @@ -244,7 +240,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of expand"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of expand"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -265,10 +261,10 @@ class VeloxMetricsApi extends MetricsApi with Logging { "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"), "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions"), "bytesSpilled" -> SQLMetrics.createSizeMetric(sparkContext, "shuffle bytes spilled"), - "splitBufferSize" -> SQLMetrics.createSizeMetric(sparkContext, "split buffer size total"), - "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to split"), - "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to spill"), - "deserializeTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime deserialize"), + "splitBufferSize" -> SQLMetrics.createSizeMetric(sparkContext, "split buffer size"), + "splitTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to split"), + "spillTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to spill"), + "deserializeTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to deserialize"), "avgReadBatchNumRows" -> SQLMetrics .createAverageMetric(sparkContext, "avg read batch num rows"), "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"), @@ -281,8 +277,8 @@ class VeloxMetricsApi extends MetricsApi with Logging { baseMetrics } else { baseMetrics ++ Map( - "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime to compress"), - "decompressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime decompress") + "compressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to compress"), + "decompressTime" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time to decompress") ) } } @@ -292,7 +288,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of window"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of window"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -314,7 +310,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { Map( "numInputRows" -> SQLMetrics.createMetric(sparkContext, "number of input rows"), "numOutputBatches" -> SQLMetrics.createMetric(sparkContext, "number of output batches"), - "convertTime" -> SQLMetrics.createTimingMetric(sparkContext, "totaltime to convert") + "convertTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to convert") ) override def genLimitTransformerMetrics(sparkContext: SparkContext): Map[String, SQLMetric] = @@ -322,7 +318,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of limit"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of limit"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -347,7 +343,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of sort"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of sort"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -370,7 +366,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "numOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "numOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of merge join"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of merge join"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -381,19 +377,19 @@ class VeloxMetricsApi extends MetricsApi with Logging { "stream preProject cpu wall time count"), "streamPreProjectionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of stream preProjection"), + "time of stream preProjection"), "bufferPreProjectionCpuCount" -> SQLMetrics.createMetric( sparkContext, "buffer preProject cpu wall time count"), "bufferPreProjectionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of buffer preProjection"), + "time of buffer preProjection"), "postProjectionCpuCount" -> SQLMetrics.createMetric( sparkContext, "postProject cpu wall time count"), "postProjectionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of postProjection") + "time of postProjection") ) override def genSortMergeJoinTransformerMetricsUpdater( @@ -433,9 +429,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "hashBuildCpuCount" -> SQLMetrics.createMetric( sparkContext, "hash build cpu wall time count"), - "hashBuildWallNanos" -> SQLMetrics.createNanoTimingMetric( - sparkContext, - "totaltime of hash build"), + "hashBuildWallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of hash build"), "hashBuildPeakMemoryBytes" -> SQLMetrics.createSizeMetric( sparkContext, "hash build peak memory bytes"), @@ -469,9 +463,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "hashProbeCpuCount" -> SQLMetrics.createMetric( sparkContext, "hash probe cpu wall time count"), - "hashProbeWallNanos" -> SQLMetrics.createNanoTimingMetric( - sparkContext, - "totaltime of hash probe"), + "hashProbeWallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of hash probe"), "hashProbePeakMemoryBytes" -> SQLMetrics.createSizeMetric( sparkContext, "hash probe peak memory bytes"), @@ -501,19 +493,19 @@ class VeloxMetricsApi extends MetricsApi with Logging { "stream preProject cpu wall time count"), "streamPreProjectionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of stream preProjection"), + "time of stream preProjection"), "buildPreProjectionCpuCount" -> SQLMetrics.createMetric( sparkContext, "preProject cpu wall time count"), "buildPreProjectionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime to build preProjection"), + "time to build preProjection"), "postProjectionCpuCount" -> SQLMetrics.createMetric( sparkContext, "postProject cpu wall time count"), "postProjectionWallNanos" -> SQLMetrics.createNanoTimingMetric( sparkContext, - "totaltime of postProjection"), + "time of postProjection"), "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "numOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "numOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes") @@ -528,9 +520,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric( - sparkContext, - "total time of NestedLoopJoin"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of NestedLoopJoin"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( @@ -546,7 +536,7 @@ class VeloxMetricsApi extends MetricsApi with Logging { "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "outputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "outputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of sample"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of sample"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala index c9b0abd6fabf..8ceea8c14f6a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala @@ -55,7 +55,7 @@ case class GenerateExecTransformer( "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"), "numOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of output vectors"), "numOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of output bytes"), - "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "totaltime of generate"), + "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of generate"), "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"), "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory bytes"), "numMemoryAllocations" -> SQLMetrics.createMetric( From f8b1968ea5dd1538a49183f74a1583d468347b39 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Thu, 13 Jun 2024 11:54:25 +0800 Subject: [PATCH 259/402] [VL] Change VLOG to DLOG in shuffle to fix performance issue in corner cases --- cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc index 741ca8ab9b40..3bd1a2fbc6cc 100644 --- a/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxHashBasedShuffleWriter.cc @@ -858,8 +858,8 @@ uint32_t VeloxHashBasedShuffleWriter::calculatePartitionBufferSize( memLimit > 0 && bytesPerRow > 0 ? memLimit / bytesPerRow / numPartitions_ >> 2 : options_.bufferSize; preAllocRowCnt = std::min(preAllocRowCnt, (uint64_t)options_.bufferSize); - VLOG(9) << "Calculated partition buffer size - memLimit: " << memLimit << ", bytesPerRow: " << bytesPerRow - << ", preAllocRowCnt: " << preAllocRowCnt << std::endl; + DLOG(INFO) << "Calculated partition buffer size - memLimit: " << memLimit << ", bytesPerRow: " << bytesPerRow + << ", preAllocRowCnt: " << preAllocRowCnt << std::endl; VS_PRINTLF(preAllocRowCnt); @@ -1400,7 +1400,7 @@ arrow::Result VeloxHashBasedShuffleWriter::partitionBufferSizeAfterShr arrow::Status VeloxHashBasedShuffleWriter::preAllocPartitionBuffers(uint32_t preAllocBufferSize) { for (auto& pid : partitionUsed_) { auto newSize = std::max(preAllocBufferSize, partition2RowCount_[pid]); - VLOG_IF(9, partitionBufferSize_[pid] != newSize) + DLOG_IF(INFO, partitionBufferSize_[pid] != newSize) << "Actual partition buffer size - current: " << partitionBufferSize_[pid] << ", newSize: " << newSize << std::endl; // Make sure the size to be allocated is larger than the size to be filled. From 1f51cf7f3d2efa03694a19eec238cf2db15b66cd Mon Sep 17 00:00:00 2001 From: Jacky Lee Date: Thu, 13 Jun 2024 14:00:20 +0800 Subject: [PATCH 260/402] [VL][BUILD] Improve compilation speed for Arrow (#6061) --- ep/build-velox/src/build_velox.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 5d9eba904480..0224e9546861 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -285,8 +285,8 @@ function compile_arrow_java_module() { # Because arrow-bom module need the -DprocessAllModules mvn versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules - mvn clean install -am \ - -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip + mvn clean install -pl bom,maven/module-info-compiler-maven-plugin,vector -am \ + -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly # Arrow C Data Interface CPP libraries mvn generate-resources -P generate-libs-cdata-all-os -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \ @@ -301,7 +301,7 @@ function compile_arrow_java_module() { # Arrow Java libraries mvn install -Parrow-jni -P arrow-c-data -pl c,dataset -am \ -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.cpp.build.dir=$ARROW_INSTALL_DIR/lib \ - -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip + -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly popd } From 00fee1d69b4b2810b95ec2215e32d9742c522e74 Mon Sep 17 00:00:00 2001 From: JiaKe Date: Thu, 13 Jun 2024 15:37:41 +0800 Subject: [PATCH 261/402] [VL] Daily Update Velox Version (2024_06_13) (#6070) --- cpp/velox/shuffle/VeloxShuffleWriter.h | 2 +- ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.h b/cpp/velox/shuffle/VeloxShuffleWriter.h index 104b87616291..7318867fc590 100644 --- a/cpp/velox/shuffle/VeloxShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxShuffleWriter.h @@ -50,7 +50,7 @@ class VeloxShuffleWriter : public ShuffleWriter { public: facebook::velox::RowVectorPtr getStrippedRowVector(const facebook::velox::RowVector& rv) { // get new row type - auto rowType = rv.type()->asRow(); + auto& rowType = rv.type()->asRow(); auto typeChildren = rowType.children(); typeChildren.erase(typeChildren.begin()); auto newRowType = facebook::velox::ROW(std::move(typeChildren)); diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 925a09630bb2..5840f21c251b 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_12 +VELOX_BRANCH=2024_06_13 VELOX_HOME="" #Set on run gluten on HDFS From 142cf0fbca70d4de386999ab7b358cb07a2566ec Mon Sep 17 00:00:00 2001 From: Chang chen Date: Thu, 13 Jun 2024 19:44:30 +0800 Subject: [PATCH 262/402] [CH][UT] Fix UT due to https://github.com/ClickHouse/ClickHouse/pull/64427 (#6079) Don't know the reason, but this PR changed the 'input_format_parquet_max_block_size' to 'DEFAULT_block_size', which is 65409, causing the MergeTree related tests to fail. Let's set input_format_parquet_max_block_size to 8192, which reverts 7445f02c9414c98956d1ad4d9318f6e545e1740c Also I use assertResult instead of assert, so we can know the actual result once failed. --- ...utenClickHouseMergeTreeOptimizeSuite.scala | 149 ++++---- ...ickHouseMergeTreePathBasedWriteSuite.scala | 311 ++++++++-------- .../GlutenClickHouseMergeTreeWriteSuite.scala | 335 ++++++++---------- .../GlutenClickHouseTableAfterRestart.scala | 24 +- 4 files changed, 377 insertions(+), 442 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index 6f8b4d93beb6..e76d3ca55d68 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -59,6 +59,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite .set( "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert", "false") + .set( + "spark.gluten.sql.columnar.backend.ch.runtime_settings.input_format_parquet_max_block_size", + "8192") } override protected def createTPCHNotNullTables(): Unit = { @@ -80,10 +83,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) - assert( - countFiles(new File(s"$basePath/lineitem_mergetree_optimize")) == 641 + assertResult(462)( + countFiles(new File(s"$basePath/lineitem_mergetree_optimize")) ) // many merged parts } } @@ -116,23 +119,23 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sparkContext.setJobGroup("test", "test") spark.sql("optimize lineitem_mergetree_optimize_p") val job_ids = spark.sparkContext.statusTracker.getJobIdsForGroup("test") - assert(job_ids.size == 1) // will not trigger actual merge job + assertResult(1)(job_ids.length) // will not trigger actual merge job spark.sparkContext.clearJobGroup() val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) + assertResult(22728)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p"))) spark.sql("VACUUM lineitem_mergetree_optimize_p RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22728) + assertResult(22728)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p"))) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) == 22730) + assertResult(22730)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p"))) } val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() - assert(ret2.apply(0).get(0) == 600572) + assertResult(600572)(ret2.apply(0).get(0)) } test("test mergetree optimize partitioned by one low card column") { @@ -152,33 +155,33 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p2") val job_ids = spark.sparkContext.statusTracker.getJobIdsForGroup("test2") if (sparkVersion.equals("3.2")) { - assert(job_ids.size == 7) // WILL trigger actual merge job + assertResult(7)(job_ids.length) // WILL trigger actual merge job } else { - assert(job_ids.size == 8) // WILL trigger actual merge job + assertResult(8)(job_ids.length) // WILL trigger actual merge job } spark.sparkContext.clearJobGroup() val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 348) + assertResult(372)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))) spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 236) + assertResult(239)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 238) + assertResult(241)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))) } spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS") // the second VACUUM will remove some empty folders if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 220) + assertResult(220)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2")) == 226) + assertResult(226)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))) } val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p2").collect() - assert(ret2.apply(0).get(0) == 600572) + assertResult(600572)(ret2.apply(0).get(0)) } test("test mergetree optimize partitioned by two low card column") { @@ -197,24 +200,24 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p3") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 448) + assertResult(516)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))) spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 294) + assertResult(306)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 296) + assertResult(308)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))) } spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 272) + assertResult(276)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3")) == 278) + assertResult(282)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))) } val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p3").collect() - assert(ret2.apply(0).get(0) == 600572) + assertResult(600572)(ret2.apply(0).get(0)) } } @@ -234,29 +237,29 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p4") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 448) + assertResult(516)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))) spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 294) + assertResult(306)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 296) + assertResult(308)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))) } spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 272) + assertResult(276)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))) } else { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4")) == 278) + assertResult(282)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))) } val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p4").collect() - assert(ret2.apply(0).get(0) == 600572) + assertResult(600572)(ret2.apply(0).get(0)) } } test("test mergetree optimize with optimize.minFileSize and optimize.maxFileSize") { - withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "83800000") { + withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "838000") { // 3 from 37 parts are larger than this, so after optimize there should be 4 parts: // 3 original parts and 1 merged part spark.sql(s""" @@ -275,20 +278,20 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 75) + assertResult(99)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. // this case will create a checkpoint - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 81) + assertResult(105)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))) } val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) } withSQLConf( - ("spark.databricks.delta.optimize.maxFileSize" -> "10000000"), - ("spark.databricks.delta.optimize.minFileSize" -> "838250")) { + "spark.databricks.delta.optimize.maxFileSize" -> "10000000", + "spark.databricks.delta.optimize.minFileSize" -> "838250") { // of the remaing 3 original parts, 2 are less than 838250, 1 is larger (size 838255) // the merged part is ~27MB, so after optimize there should be 3 parts: // 1 merged part from 2 original parts, 1 merged part from 34 original parts @@ -299,14 +302,14 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 75) + assertResult(93)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 85) + assertResult(104)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))) } val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) } // now merge all parts (testing merging from merged parts) @@ -315,14 +318,14 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS") if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 75) + assertResult(77)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))) } else { // For Spark 3.3 + Delta 2.3, vacuum command will create two commit files in deltalog dir. - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5")) == 90) + assertResult(93)(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))) } val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p5").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) } test("test mergetree optimize table with partition and bucket") { @@ -343,24 +346,22 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sql("optimize lineitem_mergetree_optimize_p6") val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p6").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 475 else 501 - }) + assertResult(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")))( + if (sparkVersion.equals("3.2")) 499 else 528) spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") spark.sql("VACUUM lineitem_mergetree_optimize_p6 RETAIN 0 HOURS") - assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")) == { - if (sparkVersion.equals("3.2")) 315 else 327 - }) + assertResult(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6")))( + if (sparkVersion.equals("3.2")) 315 else 327) val ret2 = spark.sql("select count(*) from lineitem_mergetree_optimize_p6").collect() - assert(ret2.apply(0).get(0) == 600572) + assertResult(600572)(ret2.apply(0).get(0)) } test("test skip index after optimize") { withSQLConf( - "spark.databricks.delta.optimize.maxFileSize" -> "100000000", + "spark.databricks.delta.optimize.maxFileSize" -> "2000000", "spark.sql.adaptive.enabled" -> "false") { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_index; @@ -385,12 +386,12 @@ class GlutenClickHouseMergeTreeOptimizeSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) - val mergetreeScan = scanExec(0) + assertResult(1)(scanExec.size) + val mergetreeScan = scanExec.head val ret = df.collect() - assert(ret.apply(0).get(0) == 2) + assertResult(2)(ret.apply(0).get(0)) val marks = mergetreeScan.metrics("selectedMarks").value - assert(marks == 1) + assertResult(1)(marks) val directory = new File(s"$basePath/lineitem_mergetree_index") val partDir = directory.listFiles().filter(f => f.getName.endsWith("merged")).head @@ -403,7 +404,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite test("test mergetree optimize with the path based table") { val dataPath = s"$basePath/lineitem_mergetree_optimize_path_based" clearDataPath(dataPath) - withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "83800000") { + withSQLConf("spark.databricks.delta.optimize.minFileSize" -> "838000") { // 3 from 37 parts are larger than this, so after optimize there should be 4 parts: // 3 original parts and 1 merged part @@ -422,18 +423,18 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(dataPath)) == 75) + assertResult(99)(countFiles(new File(dataPath))) } else { - assert(countFiles(new File(dataPath)) == 81) + assertResult(105)(countFiles(new File(dataPath))) } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) } withSQLConf( - ("spark.databricks.delta.optimize.maxFileSize" -> "10000000"), - ("spark.databricks.delta.optimize.minFileSize" -> "838250")) { + "spark.databricks.delta.optimize.maxFileSize" -> "10000000", + "spark.databricks.delta.optimize.minFileSize" -> "838250") { // of the remaing 3 original parts, 2 are less than 838250, 1 is larger (size 838255) // the merged part is ~27MB, so after optimize there should be 3 parts: // 1 merged part from 2 original parts, 1 merged part from 34 original parts @@ -445,13 +446,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(dataPath)) == 75) + assertResult(93)(countFiles(new File(dataPath))) } else { - assert(countFiles(new File(dataPath)) == 85) + assertResult(104)(countFiles(new File(dataPath))) } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) } // now merge all parts (testing merging from merged parts) @@ -461,19 +462,19 @@ class GlutenClickHouseMergeTreeOptimizeSuite clickhouseTable.vacuum(0.0) clickhouseTable.vacuum(0.0) if (sparkVersion.equals("3.2")) { - assert(countFiles(new File(dataPath)) == 75) + assertResult(77)(countFiles(new File(dataPath))) } else { - assert(countFiles(new File(dataPath)) == 90) + assertResult(93)(countFiles(new File(dataPath))) } val ret = spark.sql(s"select count(*) from clickhouse.`$dataPath`").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) } test("test mergetree insert with optimize basic") { withSQLConf( - ("spark.databricks.delta.optimize.minFileSize" -> "200000000"), - ("spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true") + "spark.databricks.delta.optimize.minFileSize" -> "200000000", + "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true" ) { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_insert_optimize_basic; @@ -487,10 +488,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite |""".stripMargin) val ret = spark.sql("select count(*) from lineitem_mergetree_insert_optimize_basic").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) eventually(timeout(60.seconds), interval(3.seconds)) { - assert( - new File(s"$basePath/lineitem_mergetree_insert_optimize_basic").listFiles().length == 2 + assertResult(2)( + new File(s"$basePath/lineitem_mergetree_insert_optimize_basic").listFiles().length ) } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala index c8c6307aba06..791239fabf48 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala @@ -60,6 +60,9 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .set( "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert", "false") + .set( + "spark.gluten.sql.columnar.backend.ch.runtime_settings.input_format_parquet_max_block_size", + "8192") } override protected def createTPCHNotNullTables(): Unit = { @@ -143,7 +146,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite case f: FileSourceScanExecTransformer => f case w: WholeStageTransformer => w } - assert(plans.size == 4) + assertResult(4)(plans.size) val mergetreeScan = plans(3).asInstanceOf[FileSourceScanExecTransformer] assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -154,10 +157,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) // GLUTEN-5060: check the unnecessary FilterExec val wholeStageTransformer = plans(2).asInstanceOf[WholeStageTransformer] @@ -174,7 +175,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .load(dataPath) .where("l_shipdate = date'1998-09-02'") .count() - assert(result == 183) + assertResult(183)(result) } test("test mergetree path based write with dataframe api") { @@ -236,40 +237,35 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite case f: FileSourceScanExecTransformer => f case w: WholeStageTransformer => w } - assert(plans.size == 4) + assertResult(4)(plans.size) val mergetreeScan = plans(3).asInstanceOf[FileSourceScanExecTransformer] assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) - assert( + .mkString(",")) + assertResult("l_returnflag,l_linestatus")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .lowCardKeyOption .get - .mkString(",") - .equals("l_returnflag,l_linestatus")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) // GLUTEN-5060: check the unnecessary FilterExec val wholeStageTransformer = plans(2).asInstanceOf[WholeStageTransformer] @@ -286,7 +282,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .load(dataPath) .where("l_shipdate = date'1998-09-02'") .collect() - assert(result.length == 183) + assertResult(110501)(result.apply(0).get(0)) } test("test mergetree path based insert overwrite partitioned table with small table, static") { @@ -320,7 +316,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .format("clickhouse") .load(dataPath) .count() - assert(result == 2418) + assertResult(2418)(result) } test("test mergetree path based insert overwrite partitioned table with small table, dynamic") { @@ -355,7 +351,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .format("clickhouse") .load(dataPath) .count() - assert(result == 600572) + assertResult(600572)(result) } } @@ -381,11 +377,11 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .format("clickhouse") .load(dataPath) .where("l_returnflag = 'Z'") - assert(df.count() == 1) + assertResult(1)(df.count()) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -397,16 +393,13 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert( - addFiles.map(_.rows).sum - == 600572) - + assertResult(600572)(addFiles.map(_.rows).sum) // 4 parts belong to the first batch // 2 parts belong to the second batch (1 actual updated part, 1 passively updated). - assert(addFiles.size == 6) + assertResult(6)(addFiles.size) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) - assert(filePaths.size == 2) - assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assertResult(2)(filePaths.size) + assertResult(Array(2, 4))(filePaths.values.map(paths => paths.size).toArray.sorted) } val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) @@ -417,33 +410,31 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .format("clickhouse") .load(dataPath) .where("l_returnflag = 'X'") - assert(df.count() == 1) + assertResult(1)(df.count()) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(600572)(addFiles.map(_.rows).sum) // 4 parts belong to the first batch // 2 parts belong to the second batch (1 actual updated part, 1 passively updated). - assert(addFiles.size == 6) + assertResult(6)(addFiles.size) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) - assert(filePaths.size == 3) - assert(Array(1, 2, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assertResult(2)(filePaths.size) + assertResult(Array(2, 4))(filePaths.values.map(paths => paths.size).toArray.sorted) } val df = spark.read .format("clickhouse") .load(dataPath) - assert(df.count() == 600572) + assertResult(600572)(df.count()) } test("test mergetree path based table delete") { @@ -465,7 +456,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val df = spark.read .format("clickhouse") .load(dataPath) - assert(df.count() == 600571) + assertResult(600571)(df.count()) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } @@ -474,17 +465,17 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) // 4 parts belong to the first batch // 2 parts belong to the second batch (1 actual updated part, 1 passively updated). - assert(addFiles.size == 6) + assertResult(6)(addFiles.size) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) - assert(filePaths.size == 2) - assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assertResult(2)(filePaths.size) + assertResult(Array(2, 4))(filePaths.values.map(paths => paths.size).toArray.sorted) val clickhouseTable = ClickhouseTable.forPath(spark, dataPath) clickhouseTable.delete("mod(l_orderkey, 3) = 2") val df1 = spark.read .format("clickhouse") .load(dataPath) - assert(df1.count() == 400089) + assertResult(400089)(df1.count()) } test("test mergetree path based table upsert") { @@ -503,8 +494,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val df0 = spark.sql(s""" | select count(*) from clickhouse.`$dataPath` |""".stripMargin) - assert( - df0.collect().apply(0).get(0) == 600572 + assertResult(600572)( + df0.collect().apply(0).get(0) ) upsertSourceTableAndCheck(dataPath) } @@ -540,8 +531,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val df1 = spark.sql(s""" | select count(*) from clickhouse.`$dataPath` |""".stripMargin) - assert( - df1.collect().apply(0).get(0) == 600572 + 3506 + assertResult(600572 + 3506)( + df1.collect().apply(0).get(0) ) } { @@ -549,8 +540,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite spark.sql(s""" | select count(*) from clickhouse.`$dataPath` where l_returnflag = 'Z' |""".stripMargin) - assert( - df2.collect().apply(0).get(0) == 3506 + assertResult(3506)( + df2.collect().apply(0).get(0) ) } @@ -559,8 +550,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite spark.sql(s""" | select count(*) from clickhouse.`$dataPath` where l_orderkey > 10000000 |""".stripMargin) - assert( - df3.collect().apply(0).get(0) == 3506 + assertResult(3506)( + df3.collect().apply(0).get(0) ) } } @@ -610,33 +601,31 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } val df = spark.read @@ -650,7 +639,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite |""".stripMargin) .agg(sum("l_linenumber").alias("res")) val result = df.collect() - assert(result(0).getLong(0) == 34842) + assertResult(34842)(result(0).getLong(0)) } test("test mergetree path based write with partition") { @@ -707,62 +696,56 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite runTPCHQueryBySQL(1, sqlStr, compareResult = false) { df => val result = df.collect() - assert(result.size == 4) - assert(result(0).getString(0).equals("A")) - assert(result(0).getString(1).equals("F")) - assert(result(0).getDouble(2) == 3803858.0) + assertResult(4)(result.length) + assertResult("A")(result(0).getString(0)) + assertResult("F")(result(0).getString(1)) + assertResult(3803858.0)(result(0).getDouble(2)) - assert(result(2).getString(0).equals("N")) - assert(result(2).getString(1).equals("O")) - assert(result(2).getDouble(2) == 7454519.0) + assertResult("N")(result(2).getString(0)) + assertResult("O")(result(2).getString(1)) + assertResult(7454519.0)(result(2).getDouble(2)) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) - assert(mergetreeScan.metrics("numFiles").value == 3744) + assertResult(3744)(mergetreeScan.metrics("numFiles").value) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert( + .mkString(",")) + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 2) - assert( + .mkString(",")) + assertResult(2)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_shipdate")) - assert( + .partitionColumns + .head) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(1) - .equals("l_returnflag")) + .partitionColumns(1)) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 3835) - assert(addFiles.map(_.rows).sum == 602945) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 2) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 2) + assertResult(3835)(addFiles.size) + assertResult(602945)(addFiles.map(_.rows).sum) + assertResult(2)(addFiles.count(_.partitionValues("l_shipdate").equals("1992-06-01"))) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1993-01-01"))) + assertResult(2)(addFiles.count(_.partitionValues("l_shipdate").equals("1995-01-21"))) } } @@ -814,61 +797,49 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) val buckets = ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption - assert(!buckets.isEmpty) - assert(buckets.get.numBuckets == 4) - assert( + assert(buckets.isDefined) + assertResult(4)(buckets.get.numBuckets) + assertResult("l_partkey,l_returnflag")( buckets.get.sortColumnNames - .mkString(",") - .equals("l_partkey,l_returnflag")) - assert( + .mkString(",")) + assertResult("l_orderkey")( buckets.get.bucketColumnNames - .mkString(",") - .equals("l_orderkey")) - assert( + .mkString(",")) + assertResult("l_partkey,l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_partkey,l_returnflag")) - assert( + .mkString(",")) + assertResult("l_partkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_partkey")) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + .mkString(",")) + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_shipdate")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 10089) - assert(addFiles.map(_.rows).sum == 600572) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 4) - assert( - addFiles - .filter( - f => - f.partitionValues.get("l_shipdate").get.equals("1995-01-21") && f.bucketNum.equals( - "00000")) - .size == 1) + assertResult(10089)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1992-06-01"))) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1993-01-01"))) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1995-01-21"))) + assertResult(1)(addFiles.count( + f => f.partitionValues("l_shipdate").equals("1995-01-21") && f.bucketNum.equals("00000"))) } // check part pruning effect of filter on bucket column val df = spark.sql(s""" @@ -883,7 +854,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .flatMap(partition => partition.asInstanceOf[GlutenMergeTreePartition].partList) .map(_.name) .distinct - assert(touchedParts.size == 1) + assertResult(1)(touchedParts.size) // test upsert on partitioned & bucketed table upsertSourceTableAndCheck(dataPath) @@ -929,9 +900,9 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] @@ -941,10 +912,8 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } } @@ -1052,9 +1021,9 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite | |""".stripMargin - assert( + assertResult("R")( // total rows should remain unchanged - spark.sql(sqlStr2).collect().apply(0).get(0) == "R" + spark.sql(sqlStr2).collect().apply(0).get(0) ) // test select * @@ -1101,40 +1070,38 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } - assert(plans.size == 1) - assert(plans(0).metrics("selectedMarksPk").value === 15) - assert(plans(0).metrics("totalMarksPk").value === 74) + assertResult(1)(plans.size) + assertResult(17)(plans.head.metrics("selectedMarksPk").value) + assertResult(74)(plans.head.metrics("totalMarksPk").value) } } @@ -1161,12 +1128,12 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) - val mergetreeScan = scanExec(0) + assertResult(1)(scanExec.size) + val mergetreeScan = scanExec.head - assert(ret.apply(0).get(0) == 1) + assertResult(1)(ret.apply(0).get(0)) val marks = mergetreeScan.metrics("selectedMarks").value - assert(marks == 1) + assertResult(1)(marks) val directory = new File(dataPath) // find a folder whose name is like 48b70783-b3b8-4bf8-9c52-5261aead8e3e_0_006 @@ -1197,11 +1164,11 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) - val mergetreeScan = scanExec(0) - assert(ret.apply(0).get(0) == 2) + assertResult(1)(scanExec.size) + val mergetreeScan = scanExec.head + assertResult(2)(ret.apply(0).get(0)) val marks = mergetreeScan.metrics("selectedMarks").value - assert(marks == 2) + assertResult(1)(marks) val directory = new File(dataPath) // find a folder whose name is like 48b70783-b3b8-4bf8-9c52-5261aead8e3e_0_006 @@ -1233,11 +1200,11 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) - val mergetreeScan = scanExec(0) - assert(ret.apply(0).get(0) == 2) + assertResult(1)(scanExec.size) + val mergetreeScan = scanExec.head + assertResult(2)(ret.apply(0).get(0)) val marks = mergetreeScan.metrics("selectedMarks").value - assert(marks == 1) + assertResult(1)(marks) val directory = new File(dataPath) // find a folder whose name is like 48b70783-b3b8-4bf8-9c52-5261aead8e3e_0_006 @@ -1277,18 +1244,16 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } } @@ -1320,7 +1285,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val fileFilter = new WildcardFileFilter("*_0_*") var dataFileList = dataPathFile.list(fileFilter) - assert(dataFileList.size == 5) + assertResult(6)(dataFileList.length) // re-create the same table val dataPath2 = s"$basePath/lineitem_mergetree_5219_s" @@ -1339,7 +1304,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assert(dataPathFile.isDirectory && dataPathFile.isDirectory) dataFileList = dataPathFile.list(fileFilter) - assert(dataFileList.size == 5) + assertResult(6)(dataFileList.length) } } // scalastyle:off line.size.limit diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala index 679cea37ba67..70c6553416e2 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala @@ -57,6 +57,9 @@ class GlutenClickHouseMergeTreeWriteSuite .set( "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert", "false") + .set( + "spark.gluten.sql.columnar.backend.ch.runtime_settings.input_format_parquet_max_block_size", + "8192") } override protected def createTPCHNotNullTables(): Unit = { @@ -128,7 +131,7 @@ class GlutenClickHouseMergeTreeWriteSuite case f: FileSourceScanExecTransformer => f case w: WholeStageTransformer => w } - assert(plans.size == 4) + assertResult(4)(plans.size) val mergetreeScan = plans(3).asInstanceOf[FileSourceScanExecTransformer] assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -140,10 +143,8 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) // GLUTEN-5060: check the unnecessary FilterExec val wholeStageTransformer = plans(2).asInstanceOf[WholeStageTransformer] @@ -200,9 +201,9 @@ class GlutenClickHouseMergeTreeWriteSuite | select count(*) from lineitem_mergetree_insertoverwrite | |""".stripMargin - assert( + assertResult(300001)( // total rows should remain unchanged - spark.sql(sql2).collect().apply(0).get(0) == 300001 + spark.sql(sql2).collect().apply(0).get(0) ) } @@ -251,9 +252,9 @@ class GlutenClickHouseMergeTreeWriteSuite | select count(*) from lineitem_mergetree_insertoverwrite2 | |""".stripMargin - assert( + assertResult(2418)( // total rows should remain unchanged - spark.sql(sql2).collect().apply(0).get(0) == 2418 + spark.sql(sql2).collect().apply(0).get(0) ) } @@ -303,9 +304,9 @@ class GlutenClickHouseMergeTreeWriteSuite | select count(*) from lineitem_mergetree_insertoverwrite3 | |""".stripMargin - assert( + assertResult(600572)( // total rows should remain unchanged - spark.sql(sql2).collect().apply(0).get(0) == 600572 + spark.sql(sql2).collect().apply(0).get(0) ) } } @@ -357,14 +358,14 @@ class GlutenClickHouseMergeTreeWriteSuite val df = spark.sql(sql1) val result = df.collect() - assert( + assertResult(1)( // in test data, there are only 1 row with l_orderkey = 12647 - result.apply(0).get(0) == 1 + result.apply(0).get(0) ) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -376,16 +377,14 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(600572)(addFiles.map(_.rows).sum) // 4 parts belong to the first batch // 2 parts belong to the second batch (1 actual updated part, 1 passively updated). - assert(addFiles.size == 6) + assertResult(6)(addFiles.size) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) - assert(filePaths.size == 2) - assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assertResult(2)(filePaths.size) + assertResult(Array(2, 4))(filePaths.values.map(paths => paths.size).toArray.sorted) } val sql2 = @@ -393,9 +392,9 @@ class GlutenClickHouseMergeTreeWriteSuite | select count(*) from lineitem_mergetree_update | |""".stripMargin - assert( + assertResult(600572)( // total rows should remain unchanged - spark.sql(sql2).collect().apply(0).get(0) == 600572 + spark.sql(sql2).collect().apply(0).get(0) ) } @@ -444,8 +443,8 @@ class GlutenClickHouseMergeTreeWriteSuite | select count(*) from lineitem_mergetree_delete |""".stripMargin) val result = df.collect() - assert( - result.apply(0).get(0) == 600571 + assertResult(600571)( + result.apply(0).get(0) ) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f @@ -455,10 +454,10 @@ class GlutenClickHouseMergeTreeWriteSuite val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) // 4 parts belong to the first batch // 2 parts belong to the second batch (1 actual updated part, 1 passively updated). - assert(addFiles.size == 6) + assertResult(6)(addFiles.size) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) - assert(filePaths.size == 2) - assert(Array(3, 3).sameElements(filePaths.values.map(paths => paths.size).toArray.sorted)) + assertResult(2)(filePaths.size) + assertResult(Array(2, 4))(filePaths.values.map(paths => paths.size).toArray.sorted) } { @@ -468,9 +467,7 @@ class GlutenClickHouseMergeTreeWriteSuite val df3 = spark.sql(s""" | select count(*) from lineitem_mergetree_delete |""".stripMargin) - assert( - df3.collect().apply(0).get(0) == 400089 - ) + assertResult(400089)(df3.collect().apply(0).get(0)) } } @@ -512,9 +509,7 @@ class GlutenClickHouseMergeTreeWriteSuite val df0 = spark.sql(s""" | select count(*) from lineitem_mergetree_upsert |""".stripMargin) - assert( - df0.collect().apply(0).get(0) == 600572 - ) + assertResult(600572)(df0.collect().apply(0).get(0)) } upsertSourceTableAndCheck("lineitem_mergetree_upsert") @@ -551,18 +546,14 @@ class GlutenClickHouseMergeTreeWriteSuite val df1 = spark.sql(s""" | select count(*) from $tableName |""".stripMargin) - assert( - df1.collect().apply(0).get(0) == 600572 + 3506 - ) + assertResult(600572 + 3506)(df1.collect().apply(0).get(0)) } { val df2 = spark.sql(s""" | select count(*) from $tableName where l_returnflag = 'Z' |""".stripMargin) - assert( - df2.collect().apply(0).get(0) == 3506 - ) + assertResult(3506)(df2.collect().apply(0).get(0)) } { @@ -570,9 +561,7 @@ class GlutenClickHouseMergeTreeWriteSuite spark.sql(s""" | select count(*) from $tableName where l_orderkey > 10000000 |""".stripMargin) - assert( - df3.collect().apply(0).get(0) == 3506 - ) + assertResult(3506)(df3.collect().apply(0).get(0)) } } @@ -642,33 +631,31 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } } @@ -800,62 +787,56 @@ class GlutenClickHouseMergeTreeWriteSuite runTPCHQueryBySQL(1, sqlStr, compareResult = false) { df => val result = df.collect() - assert(result.size == 4) - assert(result(0).getString(0).equals("A")) - assert(result(0).getString(1).equals("F")) - assert(result(0).getDouble(2) == 3865234.0) + assertResult(4)(result.length) + assertResult("A")(result(0).getString(0)) + assertResult("F")(result(0).getString(1)) + assertResult(3865234.0)(result(0).getDouble(2)) - assert(result(2).getString(0).equals("N")) - assert(result(2).getString(1).equals("O")) - assert(result(2).getDouble(2) == 7454519.0) + assertResult("N")(result(2).getString(0)) + assertResult("O")(result(2).getString(1)) + assertResult(7454519.0)(result(2).getDouble(2)) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) - assert(mergetreeScan.metrics("numFiles").value == 3745) + assertResult(3745)(mergetreeScan.metrics("numFiles").value) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert( + .mkString(",")) + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 2) - assert( + .mkString(",")) + assertResult(2)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_shipdate")) - assert( + .partitionColumns + .head) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(1) - .equals("l_returnflag")) + .partitionColumns(1)) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 3836) - assert(addFiles.map(_.rows).sum == 605363) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 2) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 3) + assertResult(3836)(addFiles.size) + assertResult(605363)(addFiles.map(_.rows).sum) + assertResult(2)(addFiles.count(_.partitionValues("l_shipdate").equals("1992-06-01"))) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1993-01-01"))) + assertResult(3)(addFiles.count(_.partitionValues("l_shipdate").equals("1995-01-21"))) } } @@ -927,49 +908,40 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) - assert(!ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) + assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isDefined) if (sparkVersion.equals("3.2")) { assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).orderByKeyOption.isEmpty) } else { - assert( + assertResult("l_partkey,l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_partkey,l_returnflag")) + .mkString(",")) } assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_shipdate")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 10089) - assert(addFiles.map(_.rows).sum == 600572) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 4) - assert( - addFiles - .filter( - f => - f.partitionValues.get("l_shipdate").get.equals("1995-01-21") && f.bucketNum.equals( - "00000")) - .size == 1) + assertResult(10089)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1992-06-01"))) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1993-01-01"))) + assertResult(4)(addFiles.count(_.partitionValues("l_shipdate").equals("1995-01-21"))) + assertResult(1)(addFiles.count( + f => f.partitionValues("l_shipdate").equals("1995-01-21") && f.bucketNum.equals("00000"))) } // check part pruning effect of filter on bucket column val df = spark.sql(s""" @@ -984,7 +956,7 @@ class GlutenClickHouseMergeTreeWriteSuite .flatMap(partition => partition.asInstanceOf[GlutenMergeTreePartition].partList) .map(_.name) .distinct - assert(touchedParts.size == 1) + assertResult(1)(touchedParts.size) // test upsert on partitioned & bucketed table upsertSourceTableAndCheck("lineitem_mergetree_bucket") @@ -996,9 +968,7 @@ class GlutenClickHouseMergeTreeWriteSuite val df0 = spark.sql(s""" | select count(*) from lineitem_mergetree_bucket |""".stripMargin) - assert( - df0.collect().apply(0).get(0) == 3 - ) + assertResult(3)(df0.collect().apply(0).get(0)) } @@ -1065,40 +1035,40 @@ class GlutenClickHouseMergeTreeWriteSuite warehouse + "/" + tableName } val deletedPath = new File(deletedPathStr) - assert(deletedPath.exists() == exceptedExists) + assertResult(exceptedExists)(deletedPath.exists()) } // test non external table var tableName = "lineitem_mergetree_drop" var tableLocation = "" createAndDropTable(tableName, tableLocation) - checkTableExists(tableName, tableLocation, false) + checkTableExists(tableName, tableLocation, exceptedExists = false) // test external table tableName = "lineitem_mergetree_external_drop" - createAndDropTable(tableName, tableLocation, true) - checkTableExists(tableName, tableLocation, false) + createAndDropTable(tableName, tableLocation, isExternal = true) + checkTableExists(tableName, tableLocation, exceptedExists = false) // test table with the specified location tableName = "lineitem_mergetree_location_drop" tableLocation = basePath + "/" + tableName createAndDropTable(tableName, tableLocation) - checkTableExists(tableName, tableLocation, true) + checkTableExists(tableName, tableLocation, exceptedExists = true) tableName = "lineitem_mergetree_external_location_drop" tableLocation = basePath + "/" + tableName - createAndDropTable(tableName, tableLocation, true) - checkTableExists(tableName, tableLocation, true) + createAndDropTable(tableName, tableLocation, isExternal = true) + checkTableExists(tableName, tableLocation, exceptedExists = true) tableName = "lineitem_mergetree_location_purge" tableLocation = basePath + "/" + tableName createAndDropTable(tableName, tableLocation, purgeTable = true) - checkTableExists(tableName, tableLocation, false) + checkTableExists(tableName, tableLocation, exceptedExists = false) tableName = "lineitem_mergetree_external_location_purge" tableLocation = basePath + "/" + tableName - createAndDropTable(tableName, tableLocation, true, true) - checkTableExists(tableName, tableLocation, false) + createAndDropTable(tableName, tableLocation, isExternal = true, purgeTable = true) + checkTableExists(tableName, tableLocation, exceptedExists = false) } test("test mergetree CTAS simple") { @@ -1143,9 +1113,9 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] @@ -1155,10 +1125,8 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } } @@ -1289,9 +1257,9 @@ class GlutenClickHouseMergeTreeWriteSuite | |""".stripMargin - assert( + assertResult("R")( // total rows should remain unchanged - spark.sql(sqlStr2).collect().apply(0).get(0) == "R" + spark.sql(sqlStr2).collect().apply(0).get(0) ) // test select * @@ -1359,40 +1327,38 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } - assert(plans.size == 1) - assert(plans(0).metrics("selectedMarksPk").value === 15) - assert(plans(0).metrics("totalMarksPk").value === 74) + assertResult(1)(plans.size) + assertResult(17)(plans.head.metrics("selectedMarksPk").value) + assertResult(74)(plans.head.metrics("totalMarksPk").value) } } @@ -1447,21 +1413,20 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert( ClickHouseTableV2 .getTable(fileIndex.deltaLog) @@ -1470,15 +1435,15 @@ class GlutenClickHouseMergeTreeWriteSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } - assert(plans.size == 1) - assert(plans(0).metrics("selectedMarksPk").value === 15) - assert(plans(0).metrics("totalMarksPk").value === 74) + assertResult(1)(plans.size) + assertResult(17)(plans.head.metrics("selectedMarksPk").value) + assertResult(74)(plans.head.metrics("totalMarksPk").value) } } @@ -1527,21 +1492,21 @@ class GlutenClickHouseMergeTreeWriteSuite runSql(sqlStr)( df => { val result = df.collect() - assert(result.size == 1) - assert(result(0).getLong(0) == 10) + assertResult(1)(result.length) + assertResult(10)(result(0).getLong(0)) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 1) - assert(addFiles(0).rows == 10) + assertResult(1)(addFiles.size) + assertResult(10)(addFiles.head.rows) }) } @@ -1585,16 +1550,16 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert( - (addFiles.map(_.marks).sum - addFiles.size) == mergetreeScan.metrics("totalMarksPk").value) - assert(mergetreeScan.metrics("selectedMarksPk").value == exceptedCnt) + assertResult(mergetreeScan.metrics("totalMarksPk").value)( + addFiles.map(_.marks).sum - addFiles.size) + assertResult(exceptedCnt)(mergetreeScan.metrics("selectedMarksPk").value) } val sqlStr1 = @@ -1609,8 +1574,8 @@ class GlutenClickHouseMergeTreeWriteSuite runSql(sqlStr1)( df => { val result = df.collect() - assert(result.size == 1) - assert(result(0).getDouble(0).toString.substring(0, 6).equals("2.6480")) + assertResult(1)(result.length) + assertResult("2.6480")(result(0).getDouble(0).toString.substring(0, 6)) checkSelectedMarksCnt(df, 34) }) @@ -1627,10 +1592,10 @@ class GlutenClickHouseMergeTreeWriteSuite runSql(sqlStr2)( df => { val result = df.collect() - assert(result.size == 1) - assert(result(0).getDouble(0).toString.substring(0, 6).equals("5.3379")) + assertResult(1)(result.length) + assertResult("5.3379")(result(0).getDouble(0).toString.substring(0, 6)) - checkSelectedMarksCnt(df, 24) + checkSelectedMarksCnt(df, 29) }) } @@ -1666,18 +1631,16 @@ class GlutenClickHouseMergeTreeWriteSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 5) - assert( - addFiles.map(_.rows).sum - == 600572) + assertResult(6)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } } @@ -1715,7 +1678,7 @@ class GlutenClickHouseMergeTreeWriteSuite val fileFilter = new WildcardFileFilter("*_0_*") var dataFileList = dataPath.list(fileFilter) - assert(dataFileList.size == 5) + assertResult(6)(dataFileList.length) // test with the normal table spark.sql(s""" @@ -1796,7 +1759,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(dataPath.isDirectory && dataPath.isDirectory) dataFileList = dataPath.list(fileFilter) - assert(dataFileList.size == 5) + assertResult(6)(dataFileList.length) // re-create the same table for (i <- 0 until 10) { @@ -1818,7 +1781,7 @@ class GlutenClickHouseMergeTreeWriteSuite assert(dataPath.isDirectory && dataPath.isDirectory) dataFileList = dataPath.list(fileFilter) - assert(dataFileList.size == 5) + assertResult(6)(dataFileList.length) } test("test mergetree with primary keys filter pruning by driver") { @@ -1872,22 +1835,22 @@ class GlutenClickHouseMergeTreeWriteSuite Seq(("true", 2), ("false", 3)).foreach( conf => { withSQLConf( - ("spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> conf._1)) { + "spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> conf._1) { runTPCHQueryBySQL(6, sqlStr) { df => val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } - assert(plans.size == 1) - assert(plans(0).getSplitInfos.size == conf._2) + assertResult(1)(plans.size) + assertResult(conf._2)(plans.head.getSplitInfos.size) } } }) @@ -1990,14 +1953,14 @@ class GlutenClickHouseMergeTreeWriteSuite Seq(("true", 2), ("false", 2)).foreach( conf => { withSQLConf( - ("spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> conf._1)) { + "spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> conf._1) { runTPCHQueryBySQL(12, sqlStr) { df => val scanExec = collect(df.queryExecution.executedPlan) { case f: BasicScanExecTransformer => f } - assert(scanExec.size == 2) - assert(scanExec(1).getSplitInfos.size == conf._2) + assertResult(2)(scanExec.size) + assertResult(conf._2)(scanExec(1).getSplitInfos.size) } } }) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala index ab11c1e0c201..f9e831cb4aa7 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala @@ -61,6 +61,9 @@ class GlutenClickHouseTableAfterRestart .set( "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert", "false") + .set( + "spark.gluten.sql.columnar.backend.ch.runtime_settings.input_format_parquet_max_block_size", + "8192") } override protected def createTPCHNotNullTables(): Unit = { @@ -180,9 +183,9 @@ class GlutenClickHouseTableAfterRestart // for this run, missing count should not increase runTPCHQueryBySQL(1, sqlStr)(_ => {}) val stats1 = ClickhouseSnapshot.deltaScanCache.stats() - assert(stats1.missCount() - oldMissingCount1 == 0) + assertResult(oldMissingCount1)(stats1.missCount()) val stats2 = ClickhouseSnapshot.addFileToAddMTPCache.stats() - assert(stats2.missCount() - oldMissingCount2 == 0) + assertResult(oldMissingCount2)(stats2.missCount()) } val oldMissingCount1 = ClickhouseSnapshot.deltaScanCache.stats().missCount() @@ -194,10 +197,9 @@ class GlutenClickHouseTableAfterRestart // after restart, additionally check stats of delta scan cache val stats1 = ClickhouseSnapshot.deltaScanCache.stats() - assert(stats1.missCount() - oldMissingCount1 == 1) + assertResult(oldMissingCount1 + 1)(stats1.missCount()) val stats2 = ClickhouseSnapshot.addFileToAddMTPCache.stats() - assert(stats2.missCount() - oldMissingCount2 == 5) - + assertResult(oldMissingCount2 + 6)(stats2.missCount()) } test("test optimize after restart") { @@ -222,7 +224,8 @@ class GlutenClickHouseTableAfterRestart restartSpark() spark.sql("optimize table_restart_optimize") - assert(spark.sql("select count(*) from table_restart_optimize").collect().apply(0).get(0) == 4) + assertResult(4)( + spark.sql("select count(*) from table_restart_optimize").collect().apply(0).get(0)) } test("test vacuum after restart") { @@ -250,7 +253,8 @@ class GlutenClickHouseTableAfterRestart spark.sql("vacuum table_restart_vacuum") - assert(spark.sql("select count(*) from table_restart_vacuum").collect().apply(0).get(0) == 4) + assertResult(4)( + spark.sql("select count(*) from table_restart_vacuum").collect().apply(0).get(0)) } test("test update after restart") { @@ -276,7 +280,8 @@ class GlutenClickHouseTableAfterRestart spark.sql("update table_restart_update set name = 'tom' where id = 1") - assert(spark.sql("select count(*) from table_restart_update").collect().apply(0).get(0) == 4) + assertResult(4)( + spark.sql("select count(*) from table_restart_update").collect().apply(0).get(0)) } test("test delete after restart") { @@ -302,7 +307,8 @@ class GlutenClickHouseTableAfterRestart spark.sql("delete from table_restart_delete where where id = 1") - assert(spark.sql("select count(*) from table_restart_delete").collect().apply(0).get(0) == 2) + assertResult(2)( + spark.sql("select count(*) from table_restart_delete").collect().apply(0).get(0)) } test("test drop after restart") { From f1bb1d678f0056a3b346b889dc9317351e0c0c6f Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 14 Jun 2024 08:53:08 +0800 Subject: [PATCH 263/402] [CI] Add CMake format check (#5941) --- .github/workflows/check_license.yml | 34 ++ .github/workflows/code_style.yml | 49 +- {dev => .github/workflows/util}/check.py | 0 {dev => .github/workflows/util}/check.sh | 4 +- .../util}/excluded_copyright_files.txt | 1 + .../workflows/util}/license-header.py | 0 .../workflows/util}/license.header | 0 {dev => .github/workflows/util}/util.py | 0 cpp-ch/CMakeLists.txt | 160 ++--- cpp-ch/local-engine/CMakeLists.txt | 208 +++---- cpp-ch/local-engine/Functions/CMakeLists.txt | 71 +-- .../Storages/Parquet/CMakeLists.txt | 15 +- .../Storages/SubstraitSource/CMakeLists.txt | 32 +- cpp-ch/local-engine/examples/CMakeLists.txt | 5 +- cpp-ch/local-engine/proto/CMakeLists.txt | 47 +- cpp-ch/local-engine/tests/CMakeLists.txt | 132 +++-- cpp/CMake/BuildGTest.cmake | 9 +- cpp/CMake/BuildGflags.cmake | 6 +- cpp/CMake/BuildGlog.cmake | 12 +- cpp/CMake/BuildGoogleBenchmark.cmake | 15 +- cpp/CMake/BuildMemkind.cmake | 121 ++-- cpp/CMake/BuildQATZstd.cmake | 80 +-- cpp/CMake/BuildQATzip.cmake | 87 +-- cpp/CMake/BuildQpl.cmake | 77 ++- cpp/CMake/ConfigArrow.cmake | 26 +- cpp/CMake/FindThrift.cmake | 103 ++-- cpp/CMake/FindZstd.cmake | 45 +- cpp/CMake/Findglog.cmake | 50 +- cpp/CMake/Findjemalloc_pic.cmake | 40 +- cpp/CMakeLists.txt | 54 +- cpp/core/CMakeLists.txt | 273 ++++----- cpp/core/benchmarks/CMakeLists.txt | 3 +- cpp/velox/CMakeLists.txt | 549 ++++++++++++------ cpp/velox/benchmarks/CMakeLists.txt | 11 +- cpp/velox/tests/CMakeLists.txt | 51 +- docs/developers/CppCodingStyle.md | 11 +- 36 files changed, 1362 insertions(+), 1019 deletions(-) create mode 100644 .github/workflows/check_license.yml rename {dev => .github/workflows/util}/check.py (100%) rename {dev => .github/workflows/util}/check.sh (90%) rename {dev => .github/workflows/util}/excluded_copyright_files.txt (68%) rename {dev => .github/workflows/util}/license-header.py (100%) rename {dev => .github/workflows/util}/license.header (100%) rename {dev => .github/workflows/util}/util.py (100%) diff --git a/.github/workflows/check_license.yml b/.github/workflows/check_license.yml new file mode 100644 index 000000000000..338397dbd6cb --- /dev/null +++ b/.github/workflows/check_license.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: license header check +on: + pull_request +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + license-check: + name: License Header Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check License Header + run: | + git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}} + pip install regex + cd $GITHUB_WORKSPACE/ + ./.github/workflows/util/check.sh ${{github.event.pull_request.base.sha}} diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index f1c9c2548324..8417264373a0 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -14,43 +14,56 @@ # limitations under the License. name: Code style checks - on: - pull_request - + pull_request: + paths: + - '.github/workflows/code_style.yml' + - 'cpp/**' + - 'cpp-ch/**' concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: - formatting-check: - name: Formatting Check + CPP-format-check: + name: CPP Format Check runs-on: ubuntu-latest strategy: matrix: path: - - check: 'cpp/core' - exclude: '' - - check: 'cpp/velox' + - check: 'cpp' exclude: '' steps: - uses: actions/checkout@v4 - - name: Run clang-format style check for C/C++ programs. + - name: Run clang-format style check for C/C++ uses: jidicula/clang-format-action@v4.11.0 with: clang-format-version: '15' check-path: ${{ matrix.path['check'] }} fallback-style: 'Google' # optional - license-check: - name: License Header Check + CMake-format-check: + name: CMake Format Check runs-on: ubuntu-latest - + container: ubuntu:22.04 steps: - - uses: actions/checkout@v3 - - - name: Check License Header + - name: Install tools + run: | + apt update -y + apt install git python3-pip -y + pip3 install --user cmake-format + - uses: actions/checkout@v4 + - name: Check CMake format run: | - git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}} - pip install regex - dev/check.sh ${{github.event.pull_request.base.sha}} + git config --global --add safe.directory $GITHUB_WORKSPACE + cd $GITHUB_WORKSPACE/ + fileList=$(find ./cpp ./cpp-ch -name CMakeLists.txt -o -name *.cmake) + for file in $fileList; do + /github/home/.local/bin/cmake-format --first-comment-is-literal True --in-place $file + done + if [ -n "$(git status --porcelain)" ]; then + echo "Please use cmake-format to format cmake files or apply the below patch." + git diff -- '*CMakeLists.txt' '*.cmake' + exit 1 + fi + echo "No CMake format issue." diff --git a/dev/check.py b/.github/workflows/util/check.py similarity index 100% rename from dev/check.py rename to .github/workflows/util/check.py diff --git a/dev/check.sh b/.github/workflows/util/check.sh similarity index 90% rename from dev/check.sh rename to .github/workflows/util/check.sh index 9b940845f92d..d8db8bd402c3 100755 --- a/dev/check.sh +++ b/.github/workflows/util/check.sh @@ -15,9 +15,9 @@ # limitations under the License. export BASE_COMMIT=$1 -dev/check.py header branch +./.github/workflows/util/check.py header branch if [ $? -ne 0 ]; then - dev/check.py header branch --fix + ./.github/workflows/util/check.py header branch --fix echo -e "\n==== Apply using:" echo "patch -p1 \< 5 min. Therefore, the smoke build ("FastTest") in CI initializes only the set of - # submodules minimally needed for a build and we cannot assume here that all submodules are populated. - message(ERROR "clickhouse ${CH_SOURCE_DIR} is missing or empty. to fix try run:") - message(STATUS " git clone --recursive --depth 1 https://github.com/Kyligence/ClickHouse.git ${CMAKE_SOURCE_DIR}") - endif() + if(NOT clickhouse_files) + # Checking out *all* submodules takes > 5 min. Therefore, the smoke build + # ("FastTest") in CI initializes only the set of submodules minimally needed + # for a build and we cannot assume here that all submodules are populated. + message(ERROR + "clickhouse ${CH_SOURCE_DIR} is missing or empty. to fix try run:") + message( + STATUS + " git clone --recursive --depth 1 https://github.com/Kyligence/ClickHouse.git ${CMAKE_SOURCE_DIR}" + ) + endif() endif() -if (EXISTS "${CH_SOURCE_DIR}/utils/extern-local-engine") - execute_process(COMMAND rm -rf ${CH_SOURCE_DIR}/utils/extern-local-engine) -endif () -execute_process(COMMAND ln -s ${CMAKE_CURRENT_SOURCE_DIR}/local-engine ${CH_SOURCE_DIR}/utils/extern-local-engine COMMAND_ERROR_IS_FATAL ANY) +if(EXISTS "${CH_SOURCE_DIR}/utils/extern-local-engine") + execute_process(COMMAND rm -rf ${CH_SOURCE_DIR}/utils/extern-local-engine) +endif() +execute_process( + COMMAND ln -s ${CMAKE_CURRENT_SOURCE_DIR}/local-engine + ${CH_SOURCE_DIR}/utils/extern-local-engine COMMAND_ERROR_IS_FATAL ANY) -# execute_process(COMMAND find ${CMAKE_CURRENT_SOURCE_DIR}/local-engine -regex '.*\.\(c\|cpp\|h\)' -exec clang-format-15 --verbose -i --style=file -i {} \;) +# execute_process(COMMAND find ${CMAKE_CURRENT_SOURCE_DIR}/local-engine -regex +# '.*\.\(c\|cpp\|h\)' -exec clang-format-15 --verbose -i --style=file -i {} \;) set(CH_BINARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build") option(ENABLE_CPP_TEST "Build CPP Unit test" OFF) -if (ENABLE_CPP_TEST) -add_custom_command( - USES_TERMINAL - COMMAND - bash -c - \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DENABLE_PROTOBUF=ON - -DENABLE_TESTS=ON - -DENABLE_JEMALLOC=ON - -DENABLE_MULTITARGET_CODE=ON - -DENABLE_EXTERN_LOCAL_ENGINE=ON - -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' - -S ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && - cmake --build ${CH_BINARY_DIR} --target ch unit_tests_local_engine\" - OUTPUT _build_ch) +if(ENABLE_CPP_TEST) + add_custom_command( + USES_TERMINAL + COMMAND + bash -c \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DENABLE_PROTOBUF=ON + -DENABLE_TESTS=ON -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON + -DENABLE_EXTERN_LOCAL_ENGINE=ON -DCOMPILER_FLAGS='-fvisibility=hidden + -fvisibility-inlines-hidden' -S ${CH_SOURCE_DIR} -G Ninja -B + ${CH_BINARY_DIR} && cmake --build ${CH_BINARY_DIR} --target ch + unit_tests_local_engine\" + OUTPUT _build_ch) else() -add_custom_command( - USES_TERMINAL - COMMAND - bash -c - \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DENABLE_PROTOBUF=ON - -DENABLE_TESTS=OFF - -DENABLE_JEMALLOC=ON - -DENABLE_MULTITARGET_CODE=ON - -DENABLE_EXTERN_LOCAL_ENGINE=ON - -DENABLE_ODBC=OFF - -DENABLE_CAPNP=OFF - -DENABLE_ROCKSDB=OFF - -DENABLE_GRPC=OFF - -DENABLE_RUST=OFF - -DENABLE_H3=OFF - -DENABLE_AMQPCPP=OFF - -DENABLE_CASSANDRA=OFF - -DENABLE_KAFKA=OFF - -DENABLE_NATS=OFF - -DENABLE_LIBPQXX=OFF - -DENABLE_NURAFT=OFF - -DENABLE_DATASKETCHES=OFF - -DENABLE_SQLITE=OFF - -DENABLE_S2_GEOMETRY=OFF - -DENABLE_ANNOY=OFF - -DENABLE_ULID=OFF - -DENABLE_MYSQL=OFF - -DENABLE_BCRYPT=OFF - -DENABLE_LDAP=OFF - -DENABLE_MSGPACK=OFF - -DUSE_REPLXX=OFF - -DENABLE_CLICKHOUSE_ALL=OFF - -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' - -S ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && - cmake --build ${CH_BINARY_DIR} --target libch\" - OUTPUT _build_ch) + add_custom_command( + USES_TERMINAL + COMMAND + bash -c \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DENABLE_PROTOBUF=ON + -DENABLE_TESTS=OFF -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON + -DENABLE_EXTERN_LOCAL_ENGINE=ON -DENABLE_ODBC=OFF -DENABLE_CAPNP=OFF + -DENABLE_ROCKSDB=OFF -DENABLE_GRPC=OFF -DENABLE_RUST=OFF -DENABLE_H3=OFF + -DENABLE_AMQPCPP=OFF -DENABLE_CASSANDRA=OFF -DENABLE_KAFKA=OFF + -DENABLE_NATS=OFF -DENABLE_LIBPQXX=OFF -DENABLE_NURAFT=OFF + -DENABLE_DATASKETCHES=OFF -DENABLE_SQLITE=OFF -DENABLE_S2_GEOMETRY=OFF + -DENABLE_ANNOY=OFF -DENABLE_ULID=OFF -DENABLE_MYSQL=OFF + -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF -DENABLE_MSGPACK=OFF + -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF + -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S + ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build + ${CH_BINARY_DIR} --target libch\" + OUTPUT _build_ch) endif() add_custom_target(build_ch ALL DEPENDS _build_ch) diff --git a/cpp-ch/local-engine/CMakeLists.txt b/cpp-ch/local-engine/CMakeLists.txt index 8c96c5f98f71..93ee4b8218af 100644 --- a/cpp-ch/local-engine/CMakeLists.txt +++ b/cpp-ch/local-engine/CMakeLists.txt @@ -13,22 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. - -if (ENABLE_MULTITARGET_CODE) - add_definitions(-DENABLE_MULTITARGET_CODE=1) +if(ENABLE_MULTITARGET_CODE) + add_definitions(-DENABLE_MULTITARGET_CODE=1) else() - add_definitions(-DENABLE_MULTITARGET_CODE=0) + add_definitions(-DENABLE_MULTITARGET_CODE=0) endif() -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -ffunction-sections -fdata-sections") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -ffunction-sections -fdata-sections") -set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -ffunction-sections -fdata-sections") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -ffunction-sections -fdata-sections") +set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections") -if (COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}") +if(COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) + set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}") else() - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}") -endif () + set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}") +endif() set(THRIFT_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src") @@ -37,7 +39,7 @@ include(FindJava) include(UseJava) include(FindJNI) -#set(JNI_NATIVE_SOURCES local_engine_jni.cpp) +# set(JNI_NATIVE_SOURCES local_engine_jni.cpp) set(LOCALENGINE_SHARED_LIB _gluten_ch) add_subdirectory(proto) @@ -61,116 +63,124 @@ add_headers_and_sources(disks Disks) add_headers_and_sources(disks Disks/ObjectStorages) include_directories( - ${JNI_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR}/proto - ${THRIFT_INCLUDE_DIR} - ${CMAKE_BINARY_DIR}/contrib/thrift-cmake - ${CMAKE_BINARY_DIR}/contrib/llvm-project/llvm/include - ${CMAKE_CURRENT_SOURCE_DIR} - ${ClickHouse_SOURCE_DIR}/src - ${ClickHouse_SOURCE_DIR}/base - ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include - ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include - ${CMAKE_BINARY_DIR}/contrib/orc/c++/include - ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-blobs/inc - ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/core/azure-core/inc - ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-common/inc - ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/llvm/include - ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/utils/bazel/llvm-project-overlay/llvm/include + ${JNI_INCLUDE_DIRS} + ${CMAKE_CURRENT_BINARY_DIR}/proto + ${THRIFT_INCLUDE_DIR} + ${CMAKE_BINARY_DIR}/contrib/thrift-cmake + ${CMAKE_BINARY_DIR}/contrib/llvm-project/llvm/include + ${CMAKE_CURRENT_SOURCE_DIR} + ${ClickHouse_SOURCE_DIR}/src + ${ClickHouse_SOURCE_DIR}/base + ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include + ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include + ${CMAKE_BINARY_DIR}/contrib/orc/c++/include + ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-blobs/inc + ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/core/azure-core/inc + ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-common/inc + ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/llvm/include + ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/utils/bazel/llvm-project-overlay/llvm/include ) add_subdirectory(Storages/Parquet) add_subdirectory(Storages/SubstraitSource) add_subdirectory(Functions) -add_library(gluten_clickhouse_backend_libs - ${builder_sources} - ${join_sources} - ${parser_sources} - ${rewriter_sources} - ${storages_sources} - ${common_sources} - ${external_sources} - ${shuffle_sources} - ${operator_sources} - ${aggregate_functions_sources} - ${jni_sources} - ${disks_sources} -) - -target_link_libraries(gluten_clickhouse_backend_libs PUBLIC - substrait_source - clickhouse_aggregate_functions - clickhouse_functions - gluten_spark_functions - ch_contrib::xxHash -) +add_library( + gluten_clickhouse_backend_libs + ${builder_sources} + ${join_sources} + ${parser_sources} + ${rewriter_sources} + ${storages_sources} + ${common_sources} + ${external_sources} + ${shuffle_sources} + ${operator_sources} + ${aggregate_functions_sources} + ${jni_sources} + ${disks_sources}) + +target_link_libraries( + gluten_clickhouse_backend_libs + PUBLIC substrait_source clickhouse_aggregate_functions clickhouse_functions + gluten_spark_functions ch_contrib::xxHash) # Add udf sources files in sub directories to functions_sources -option(ENABLE_LOCAL_UDFS "Build UDFs in 'local-engine/Parser/*_udf' subdirectories" ON) -if (ENABLE_LOCAL_UDFS) - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} Parser/*_udf) - foreach(child ${children}) - add_headers_and_sources(local_udfs ${child}) - endforeach() -endif () - -file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} Parser/*_function_parser) +option(ENABLE_LOCAL_UDFS + "Build UDFs in 'local-engine/Parser/*_udf' subdirectories" ON) +if(ENABLE_LOCAL_UDFS) + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + Parser/*_udf) + foreach(child ${children}) + add_headers_and_sources(local_udfs ${child}) + endforeach() +endif() + +file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + Parser/*_function_parser) foreach(child ${children}) - add_headers_and_sources(function_parsers ${child}) + add_headers_and_sources(function_parsers ${child}) endforeach() -# Notice: soures files under Parser/*_udf subdirectories must be built into target ${LOCALENGINE_SHARED_LIB} directly -# to make sure all function parsers are registered successly. -add_library(${LOCALENGINE_SHARED_LIB} SHARED - local_engine_jni.cpp - ${local_udfs_sources} - ${function_parsers_sources} - $) # why add clickhouse_malloc? check clickhouse PR-8046 +# Notice: soures files under Parser/*_udf subdirectories must be built into +# target ${LOCALENGINE_SHARED_LIB} directly to make sure all function parsers +# are registered successly. +add_library( + ${LOCALENGINE_SHARED_LIB} SHARED + local_engine_jni.cpp ${local_udfs_sources} ${function_parsers_sources} + $) # why add clickhouse_malloc? check + # clickhouse PR-8046 target_compile_options(${LOCALENGINE_SHARED_LIB} PUBLIC -fPIC - -Wno-shorten-64-to-32) - -target_link_libraries(${LOCALENGINE_SHARED_LIB} -PUBLIC - clickhouse_new_delete - clickhouse_common_config - clickhouse_common_io - clickhouse_parsers - clickhouse_storages_system - loggers - gluten_clickhouse_backend_libs - ch_contrib::protobuf -PRIVATE - substrait -) + -Wno-shorten-64-to-32) + +target_link_libraries( + ${LOCALENGINE_SHARED_LIB} + PUBLIC clickhouse_new_delete + clickhouse_common_config + clickhouse_common_io + clickhouse_parsers + clickhouse_storages_system + loggers + gluten_clickhouse_backend_libs + ch_contrib::protobuf + PRIVATE substrait) target_link_libraries(${LOCALENGINE_SHARED_LIB} PUBLIC ch_parquet) -if (ENABLE_JEMALLOC) - target_link_options(${LOCALENGINE_SHARED_LIB} PRIVATE - -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch.map -Wl,-Bsymbolic-functions) +if(ENABLE_JEMALLOC) + target_link_options( + ${LOCALENGINE_SHARED_LIB} PRIVATE + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch.map + -Wl,-Bsymbolic-functions) else() - target_link_options(${LOCALENGINE_SHARED_LIB} PRIVATE - -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch-hide-jemalloc.map) + target_link_options( + ${LOCALENGINE_SHARED_LIB} PRIVATE + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch-hide-jemalloc.map) endif() -if ("${CMAKE_BUILD_TYPE}" MATCHES "Debug") - set(LOCALENGINE_SHARED_LIB_NAME "libchd.so") -else () - set(LOCALENGINE_SHARED_LIB_NAME "libch.so") -endif () +if("${CMAKE_BUILD_TYPE}" MATCHES "Debug") + set(LOCALENGINE_SHARED_LIB_NAME "libchd.so") +else() + set(LOCALENGINE_SHARED_LIB_NAME "libch.so") +endif() add_custom_command( - OUTPUT ${LOCALENGINE_SHARED_LIB_NAME} - COMMAND ${CMAKE_COMMAND} -E rename $ ${LOCALENGINE_SHARED_LIB_NAME} - COMMENT "Renaming $ to ${LOCALENGINE_SHARED_LIB_NAME}" - DEPENDS ${LOCALENGINE_SHARED_LIB}) + OUTPUT ${LOCALENGINE_SHARED_LIB_NAME} + COMMAND ${CMAKE_COMMAND} -E rename $ + ${LOCALENGINE_SHARED_LIB_NAME} + COMMENT + "Renaming $ to ${LOCALENGINE_SHARED_LIB_NAME}" + DEPENDS ${LOCALENGINE_SHARED_LIB}) add_custom_target(libch ALL DEPENDS ${LOCALENGINE_SHARED_LIB_NAME}) add_subdirectory(tests) -if (ENABLE_EXAMPLES) - add_subdirectory(examples) -endif() \ No newline at end of file +if(ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/cpp-ch/local-engine/Functions/CMakeLists.txt b/cpp-ch/local-engine/Functions/CMakeLists.txt index 5968c86094f7..74697315597b 100644 --- a/cpp-ch/local-engine/Functions/CMakeLists.txt +++ b/cpp-ch/local-engine/Functions/CMakeLists.txt @@ -16,51 +16,52 @@ add_headers_and_sources(gluten_spark_functions .) add_library(gluten_spark_functions_obj OBJECT ${gluten_spark_functions_sources}) -list (APPEND PRIVATE_LIBS - boost::headers_only - pcg_random - Poco::Foundation - Poco::Util - Poco::Net - Poco::JSON - ch_contrib::cctz - ch_contrib::fmt - ch_contrib::pdqsort - ch_contrib::miniselect - ch_contrib::magic_enum - ch_contrib::double_conversion - ch_contrib::dragonbox_to_chars - ch_contrib::re2 - ch_contrib::abseil_swiss_tables - ch_contrib::sparsehash - ch_contrib::metrohash - ch_contrib::murmurhash - ch_contrib::wyhash - ch_contrib::cityhash - ch_contrib::farmhash - ch_contrib::xxHash - OpenSSL::SSL -) +list( + APPEND + PRIVATE_LIBS + boost::headers_only + pcg_random + Poco::Foundation + Poco::Util + Poco::Net + Poco::JSON + ch_contrib::cctz + ch_contrib::fmt + ch_contrib::pdqsort + ch_contrib::miniselect + ch_contrib::magic_enum + ch_contrib::double_conversion + ch_contrib::dragonbox_to_chars + ch_contrib::re2 + ch_contrib::abseil_swiss_tables + ch_contrib::sparsehash + ch_contrib::metrohash + ch_contrib::murmurhash + ch_contrib::wyhash + ch_contrib::cityhash + ch_contrib::farmhash + ch_contrib::xxHash + OpenSSL::SSL) -if (TARGET ch_contrib::vectorscan) - list (APPEND PRIVATE_LIBS ch_contrib::vectorscan) +if(TARGET ch_contrib::vectorscan) + list(APPEND PRIVATE_LIBS ch_contrib::vectorscan) endif() -if (TARGET ch_contrib::rapidjson) - list (APPEND PRIVATE_LIBS ch_contrib::rapidjson) +if(TARGET ch_contrib::rapidjson) + list(APPEND PRIVATE_LIBS ch_contrib::rapidjson) endif() -if (TARGET ch_contrib::simdjson) - list (APPEND PRIVATE_LIBS ch_contrib::simdjson) +if(TARGET ch_contrib::simdjson) + list(APPEND PRIVATE_LIBS ch_contrib::simdjson) endif() -if (TARGET ch_rust::blake3) - list (APPEND PRIVATE_LIBS ch_rust::blake3) +if(TARGET ch_rust::blake3) + list(APPEND PRIVATE_LIBS ch_rust::blake3) endif() -list (APPEND OBJECT_LIBS $) +list(APPEND OBJECT_LIBS $) target_link_libraries(gluten_spark_functions_obj PRIVATE ${PRIVATE_LIBS}) add_library(gluten_spark_functions INTERFACE) -target_link_libraries(gluten_spark_functions INTERFACE ${OBJECT_LIBS}) \ No newline at end of file +target_link_libraries(gluten_spark_functions INTERFACE ${OBJECT_LIBS}) diff --git a/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt b/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt index f3d9e14f4b9c..bfe538710804 100644 --- a/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt +++ b/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt @@ -18,13 +18,10 @@ set(ARROW_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") add_headers_and_sources(Parquet .) add_library(ch_parquet ${Parquet_sources}) -target_link_libraries(ch_parquet PUBLIC - boost::headers_only - clickhouse_common_io -) +target_link_libraries(ch_parquet PUBLIC boost::headers_only + clickhouse_common_io) -target_include_directories(ch_parquet SYSTEM BEFORE PUBLIC - ${ARROW_INCLUDE_DIR} - ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src - ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src -) +target_include_directories( + ch_parquet SYSTEM BEFORE + PUBLIC ${ARROW_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src + ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src) diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt index 4e43c924f6fe..228f54255cb4 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt +++ b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt @@ -15,30 +15,24 @@ set(ARROW_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") - macro(add_headers_and_sources_including_cc prefix common_path) - add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) - add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.cc ${common_path}/*.h) + add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) + add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c + ${common_path}/*.cc ${common_path}/*.h) endmacro() add_headers_and_sources(substrait_source .) add_headers_and_sources_including_cc(ch_parquet arrow) add_library(substrait_source ${substrait_source_sources}) -target_compile_options(substrait_source PRIVATE - -Wno-suggest-destructor-override - -Wno-inconsistent-missing-destructor-override -) +target_compile_options( + substrait_source PRIVATE -Wno-suggest-destructor-override + -Wno-inconsistent-missing-destructor-override) -target_link_libraries(substrait_source PUBLIC - boost::headers_only - ch_contrib::protobuf - clickhouse_common_io - ch_contrib::hdfs - substrait -) +target_link_libraries( + substrait_source PUBLIC boost::headers_only ch_contrib::protobuf + clickhouse_common_io ch_contrib::hdfs substrait) -target_include_directories(substrait_source SYSTEM BEFORE PUBLIC - ${ARROW_INCLUDE_DIR} - ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src - ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src -) \ No newline at end of file +target_include_directories( + substrait_source SYSTEM BEFORE + PUBLIC ${ARROW_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src + ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src) diff --git a/cpp-ch/local-engine/examples/CMakeLists.txt b/cpp-ch/local-engine/examples/CMakeLists.txt index bbeeb98d2445..03cd3bfe3f19 100644 --- a/cpp-ch/local-engine/examples/CMakeLists.txt +++ b/cpp-ch/local-engine/examples/CMakeLists.txt @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -clickhouse_add_executable (signal_demo signal_demo.cpp) -target_link_libraries(signal_demo PRIVATE gluten_clickhouse_backend_libs loggers) \ No newline at end of file +clickhouse_add_executable(signal_demo signal_demo.cpp) +target_link_libraries(signal_demo PRIVATE gluten_clickhouse_backend_libs + loggers) diff --git a/cpp-ch/local-engine/proto/CMakeLists.txt b/cpp-ch/local-engine/proto/CMakeLists.txt index 31583ff659ed..ffb34504af52 100644 --- a/cpp-ch/local-engine/proto/CMakeLists.txt +++ b/cpp-ch/local-engine/proto/CMakeLists.txt @@ -12,37 +12,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -file(GLOB protobuf_files - substrait/*.proto - substrait/extensions/*.proto - ) +file(GLOB protobuf_files substrait/*.proto substrait/extensions/*.proto) -FOREACH(FIL ${protobuf_files}) - file(RELATIVE_PATH FIL_RELATIVE ${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/proto/ ${FIL}) - string(REGEX REPLACE "\\.proto" "" FILE_NAME ${FIL_RELATIVE}) - LIST(APPEND SUBSTRAIT_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.cc") - LIST(APPEND SUBSTRAIT_HEADERS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.h") -ENDFOREACH() +foreach(FIL ${protobuf_files}) + file(RELATIVE_PATH FIL_RELATIVE + ${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/proto/ ${FIL}) + string(REGEX REPLACE "\\.proto" "" FILE_NAME ${FIL_RELATIVE}) + list(APPEND SUBSTRAIT_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.cc") + list(APPEND SUBSTRAIT_HEADERS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.h") +endforeach() # Generate Substrait headers add_custom_command( - OUTPUT ${SUBSTRAIT_SRCS} ${SUBSTRAIT_HEADERS} - COMMAND - $ --cpp_out ${CMAKE_CURRENT_BINARY_DIR} - --proto_path ${CMAKE_CURRENT_SOURCE_DIR} - --proto_path ${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src - ${protobuf_files} - DEPENDS ${protobuf_files} - COMMENT "Running cpp protocol buffer compiler" - VERBATIM) -add_custom_target(generate_substrait ALL DEPENDS ${SUBSTRAIT_SRCS} ${SUBSTRAIT_HEADERS}) + OUTPUT ${SUBSTRAIT_SRCS} ${SUBSTRAIT_HEADERS} + COMMAND + $ --cpp_out ${CMAKE_CURRENT_BINARY_DIR} --proto_path + ${CMAKE_CURRENT_SOURCE_DIR} --proto_path + ${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src ${protobuf_files} + DEPENDS ${protobuf_files} + COMMENT "Running cpp protocol buffer compiler" + VERBATIM) +add_custom_target(generate_substrait ALL DEPENDS ${SUBSTRAIT_SRCS} + ${SUBSTRAIT_HEADERS}) set_source_files_properties(${SUBSTRAIT_SRCS} PROPERTIES GENERATED TRUE) add_library(substrait ${SUBSTRAIT_SRCS}) add_dependencies(substrait generate_substrait) -target_compile_options(substrait PUBLIC -fPIC - -Wno-reserved-identifier - -Wno-deprecated) -target_include_directories(substrait SYSTEM BEFORE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_compile_options(substrait PUBLIC -fPIC -Wno-reserved-identifier + -Wno-deprecated) +target_include_directories(substrait SYSTEM BEFORE + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries(substrait ch_contrib::protobuf) - diff --git a/cpp-ch/local-engine/tests/CMakeLists.txt b/cpp-ch/local-engine/tests/CMakeLists.txt index 9781a332e89c..be02bf6234d2 100644 --- a/cpp-ch/local-engine/tests/CMakeLists.txt +++ b/cpp-ch/local-engine/tests/CMakeLists.txt @@ -12,76 +12,92 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h.in ${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h) -if (ENABLE_TESTS) - macro(add_gtest_sources prefix common_path) - add_glob(${prefix}_sources ${common_path}/gtest*.cpp) - endmacro() +if(ENABLE_TESTS) + macro(add_gtest_sources prefix common_path) + add_glob(${prefix}_sources ${common_path}/gtest*.cpp) + endmacro() - set(USE_INTERNAL_GTEST_LIBRARY 0) - set(BENCHMARK_ENABLE_TESTING OFF) + set(USE_INTERNAL_GTEST_LIBRARY 0) + set(BENCHMARK_ENABLE_TESTING OFF) - enable_testing() - include(CTest) + enable_testing() + include(CTest) - include_directories(${GTEST_INCLUDE_DIRS}) + include_directories(${GTEST_INCLUDE_DIRS}) - set(TEST_DATA_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/tests") - set(HAVE_POSIX_REGEX 1) - set(LOCAL_ENGINE_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine") + set(TEST_DATA_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/tests") + set(HAVE_POSIX_REGEX 1) + set(LOCAL_ENGINE_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine") - add_gtest_sources(local_engine_gtest .) - if (ENABLE_LOCAL_UDFS) - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ../Parser/*_udf/tests) - foreach(child ${children}) - add_gtest_sources(local_engine_gtest ${child}) - endforeach() - - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ../Parser/*_udf) - foreach(child ${children}) - add_headers_and_sources(local_engine_udf ${child}) - endforeach() - endif () + add_gtest_sources(local_engine_gtest .) + if(ENABLE_LOCAL_UDFS) + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../Parser/*_udf/tests) + foreach(child ${children}) + add_gtest_sources(local_engine_gtest ${child}) + endforeach() - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ../Parser/*_function_parser) + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../Parser/*_udf) foreach(child ${children}) - add_headers_and_sources(local_engine_function_parser ${child}) + add_headers_and_sources(local_engine_udf ${child}) endforeach() + endif() + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../Parser/*_function_parser) + foreach(child ${children}) + add_headers_and_sources(local_engine_function_parser ${child}) + endforeach() - message("local engine gtest sources: ${local_engine_gtest_sources}") - message("local engine udf sources: ${local_engine_udf_sources}") - message("local engine function parser sources: ${local_engine_function_parser_sources}") + message("local engine gtest sources: ${local_engine_gtest_sources}") + message("local engine udf sources: ${local_engine_udf_sources}") + message( + "local engine function parser sources: ${local_engine_function_parser_sources}" + ) - add_executable(unit_tests_local_engine - gluten_test_util.cpp - ${local_engine_gtest_sources} - ${local_engine_udf_sources} - ${local_engine_function_parser_sources}) - target_include_directories(unit_tests_local_engine PRIVATE - ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine - ${CMAKE_CURRENT_SOURCE_DIR} - ) - # no-unreachable-code for GTEST_SKIP - target_compile_options(unit_tests_local_engine PRIVATE -Wno-unreachable-code) - target_link_libraries(unit_tests_local_engine PRIVATE gluten_clickhouse_backend_libs clickhouse_parsers loggers ch_contrib::gmock_all ch_contrib::gtest) - target_link_libraries(unit_tests_local_engine PRIVATE ch_parquet) + add_executable( + unit_tests_local_engine + gluten_test_util.cpp ${local_engine_gtest_sources} + ${local_engine_udf_sources} ${local_engine_function_parser_sources}) + target_include_directories( + unit_tests_local_engine + PRIVATE ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine + ${CMAKE_CURRENT_SOURCE_DIR}) + # no-unreachable-code for GTEST_SKIP + target_compile_options(unit_tests_local_engine PRIVATE -Wno-unreachable-code) + target_link_libraries( + unit_tests_local_engine + PRIVATE gluten_clickhouse_backend_libs clickhouse_parsers loggers + ch_contrib::gmock_all ch_contrib::gtest) + target_link_libraries(unit_tests_local_engine PRIVATE ch_parquet) endif() -if (ENABLE_BENCHMARKS) - include_directories(benchmark_local_engine SYSTEM PUBLIC - ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine - ) - add_executable(benchmark_local_engine - gluten_test_util.cpp - benchmark_local_engine.cpp - benchmark_parquet_read.cpp - benchmark_spark_row.cpp - benchmark_unix_timestamp_function.cpp - benchmark_spark_floor_function.cpp - benchmark_cast_float_function.cpp - benchmark_to_datetime_function.cpp - benchmark_spark_divide_function.cpp) - target_link_libraries(benchmark_local_engine PRIVATE gluten_clickhouse_backend_libs ch_contrib::gbenchmark_all loggers ch_parquet) +if(ENABLE_BENCHMARKS) + include_directories(benchmark_local_engine SYSTEM PUBLIC + ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine) + add_executable( + benchmark_local_engine + gluten_test_util.cpp + benchmark_local_engine.cpp + benchmark_parquet_read.cpp + benchmark_spark_row.cpp + benchmark_unix_timestamp_function.cpp + benchmark_spark_floor_function.cpp + benchmark_cast_float_function.cpp + benchmark_to_datetime_function.cpp + benchmark_spark_divide_function.cpp) + target_link_libraries( + benchmark_local_engine + PRIVATE gluten_clickhouse_backend_libs ch_contrib::gbenchmark_all loggers + ch_parquet) endif() diff --git a/cpp/CMake/BuildGTest.cmake b/cpp/CMake/BuildGTest.cmake index fff99455c419..d85578c0593c 100644 --- a/cpp/CMake/BuildGTest.cmake +++ b/cpp/CMake/BuildGTest.cmake @@ -6,15 +6,14 @@ set(GLUTEN_GTEST_BUILD_SHA256_CHECKSUM ad7fdba11ea011c1d925b3289cf4af2c66a352e18d4c7264392fead75e919363) set(GLUTEN_GTEST_SOURCE_URL "https://github.com/google/googletest/archive/refs/tags/v${GLUTEN_GTEST_VERSION}.tar.gz" - ) +) resolve_dependency_url(GTEST) message(STATUS "Building gtest from source") FetchContent_Declare( - gtest - URL ${GLUTEN_GTEST_SOURCE_URL} - URL_HASH "${GLUTEN_GTEST_BUILD_SHA256_CHECKSUM}" -) + gtest + URL ${GLUTEN_GTEST_SOURCE_URL} + URL_HASH "${GLUTEN_GTEST_BUILD_SHA256_CHECKSUM}") FetchContent_MakeAvailable(gtest) diff --git a/cpp/CMake/BuildGflags.cmake b/cpp/CMake/BuildGflags.cmake index 4cb201115835..8e66bd6b9839 100644 --- a/cpp/CMake/BuildGflags.cmake +++ b/cpp/CMake/BuildGflags.cmake @@ -17,10 +17,10 @@ include_guard(GLOBAL) set(GLUTEN_GFLAGS_BUILD_SHA256_CHECKSUM - 34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf) + 34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf) string(CONCAT GLUTEN_GFLAGS_SOURCE_URL - "https://github.com/gflags/gflags/archive/refs/tags/" - "v${GLUTEN_GFLAGS_VERSION}.tar.gz") + "https://github.com/gflags/gflags/archive/refs/tags/" + "v${GLUTEN_GFLAGS_VERSION}.tar.gz") resolve_dependency_url(GFLAGS) diff --git a/cpp/CMake/BuildGlog.cmake b/cpp/CMake/BuildGlog.cmake index cf405225c313..3f0f78a16531 100644 --- a/cpp/CMake/BuildGlog.cmake +++ b/cpp/CMake/BuildGlog.cmake @@ -14,10 +14,10 @@ include_guard(GLOBAL) set(GLUTEN_GLOG_BUILD_SHA256_CHECKSUM - 8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6) + 8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6) set(GLUTEN_GLOG_SOURCE_URL - "https://github.com/google/glog/archive/refs/tags/v${GLUTEN_GLOG_VERSION}.tar.gz" - ) + "https://github.com/google/glog/archive/refs/tags/v${GLUTEN_GLOG_VERSION}.tar.gz" +) resolve_dependency_url(GLOG) @@ -27,7 +27,7 @@ FetchContent_Declare( URL ${GLUTEN_GLOG_SOURCE_URL} URL_HASH SHA256=${GLUTEN_GLOG_BUILD_SHA256_CHECKSUM} PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/glog/glog-no-export.patch - && git apply ${CMAKE_CURRENT_LIST_DIR}/glog/glog-config.patch) + && git apply ${CMAKE_CURRENT_LIST_DIR}/glog/glog-config.patch) set(BUILD_SHARED_LIBS OFF) set(WITH_UNWIND OFF) @@ -48,6 +48,6 @@ endif() # These headers are missing from the include dir but adding the src dir causes # issues with folly so we just copy it to the include dir file(COPY ${glog_SOURCE_DIR}/src/glog/platform.h - DESTINATION ${glog_BINARY_DIR}/glog) + DESTINATION ${glog_BINARY_DIR}/glog) file(COPY ${glog_SOURCE_DIR}/src/glog/log_severity.h - DESTINATION ${glog_BINARY_DIR}/glog) + DESTINATION ${glog_BINARY_DIR}/glog) diff --git a/cpp/CMake/BuildGoogleBenchmark.cmake b/cpp/CMake/BuildGoogleBenchmark.cmake index 8efbb58eab74..a71d73432f62 100644 --- a/cpp/CMake/BuildGoogleBenchmark.cmake +++ b/cpp/CMake/BuildGoogleBenchmark.cmake @@ -21,8 +21,10 @@ include(FetchContent) set(GLUTEN_GBENCHMARK_BUILD_VERSION "v1.6.0") set(GLUTEN_GBENCHMARK_SOURCE_URL "https://github.com/google/benchmark/archive/refs/tags/${GLUTEN_GBENCHMARK_BUILD_VERSION}.tar.gz" - "https://github.com/ursa-labs/thirdparty/releases/download/latest/gbenchmark-${GLUTEN_GBENCHMARK_BUILD_VERSION}.tar.gz") -set(GLUTEN_GBENCHMARK_BUILD_SHA256_CHECKSUM "1f71c72ce08d2c1310011ea6436b31e39ccab8c2db94186d26657d41747c85d6") + "https://github.com/ursa-labs/thirdparty/releases/download/latest/gbenchmark-${GLUTEN_GBENCHMARK_BUILD_VERSION}.tar.gz" +) +set(GLUTEN_GBENCHMARK_BUILD_SHA256_CHECKSUM + "1f71c72ce08d2c1310011ea6436b31e39ccab8c2db94186d26657d41747c85d6") resolve_dependency_url(GBENCHMARK) @@ -30,12 +32,11 @@ set(GBENCHMARK_CMAKE_ARGS "-fPIC -w") message(STATUS "Building google benchmark from source") FetchContent_Declare( - gbenchmark - URL ${GLUTEN_GBENCHMARK_SOURCE_URL} - URL_HASH "${GLUTEN_GBENCHMARK_BUILD_SHA256_CHECKSUM}" -) + gbenchmark + URL ${GLUTEN_GBENCHMARK_SOURCE_URL} + URL_HASH "${GLUTEN_GBENCHMARK_BUILD_SHA256_CHECKSUM}") -if (NOT gbenchmark_POPULATED) +if(NOT gbenchmark_POPULATED) # We don't want to build tests. set(BENCHMARK_ENABLE_TESTING OFF diff --git a/cpp/CMake/BuildMemkind.cmake b/cpp/CMake/BuildMemkind.cmake index 039db0cc0d81..2f2248de6fc4 100644 --- a/cpp/CMake/BuildMemkind.cmake +++ b/cpp/CMake/BuildMemkind.cmake @@ -26,48 +26,50 @@ endif() macro(build_hwloc) message(STATUS "Building hwloc from source") set(HWLOC_BUILD_VERSION "2.8.0") - set(HWLOC_BUILD_SHA256_CHECKSUM "311d44e99bbf6d269c2cbc569d073978d88352bc31d51e31457d4df94783172d") + set(HWLOC_BUILD_SHA256_CHECKSUM + "311d44e99bbf6d269c2cbc569d073978d88352bc31d51e31457d4df94783172d") set(HWLOC_SOURCE_URL - "https://github.com/open-mpi/hwloc/archive/refs/tags/hwloc-${HWLOC_BUILD_VERSION}.tar.gz") + "https://github.com/open-mpi/hwloc/archive/refs/tags/hwloc-${HWLOC_BUILD_VERSION}.tar.gz" + ) set(HWLOC_LIB_NAME "hwloc") set(HWLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/hwloc_ep-install") set(HWLOC_SOURCE_DIR "${HWLOC_PREFIX}/src/hwloc_ep") set(HWLOC_INCLUDE_DIR "${HWLOC_SOURCE_DIR}/include") set(HWLOC_LIB_DIR "${HWLOC_SOURCE_DIR}/hwloc/.libs") - set(HWLOC_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${HWLOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(HWLOC_STATIC_LIB_NAME + "${CMAKE_STATIC_LIBRARY_PREFIX}${HWLOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) set(HWLOC_STATIC_LIB_TARGETS - "${HWLOC_SOURCE_DIR}/src/.libs/${HWLOC_STATIC_LIB_NAME}") + "${HWLOC_SOURCE_DIR}/src/.libs/${HWLOC_STATIC_LIB_NAME}") set(HWLOC_CONFIGURE_ARGS - "--prefix=${HWLOC_PREFIX}" - "--with-pic" - "--enable-static" - "--disable-shared" - "--enable-plugins") - ExternalProject_Add(hwloc_ep - PREFIX ${HWLOC_PREFIX} - URL ${HWLOC_SOURCE_URL} - URL_HASH "SHA256=${HWLOC_BUILD_SHA256_CHECKSUM}" - SOURCE_DIR ${HWLOC_SOURCE_DIR} - CONFIGURE_COMMAND ./configure ${HWLOC_CONFIGURE_ARGS} - BUILD_COMMAND ${MAKE} - BUILD_BYPRODUCTS ${HWLOC_STATIC_LIB_TARGETS} - BUILD_IN_SOURCE 1) + "--prefix=${HWLOC_PREFIX}" "--with-pic" "--enable-static" + "--disable-shared" "--enable-plugins") + ExternalProject_Add( + hwloc_ep + PREFIX ${HWLOC_PREFIX} + URL ${HWLOC_SOURCE_URL} + URL_HASH "SHA256=${HWLOC_BUILD_SHA256_CHECKSUM}" + SOURCE_DIR ${HWLOC_SOURCE_DIR} + CONFIGURE_COMMAND ./configure ${HWLOC_CONFIGURE_ARGS} + BUILD_COMMAND ${MAKE} + BUILD_BYPRODUCTS ${HWLOC_STATIC_LIB_TARGETS} + BUILD_IN_SOURCE 1) - ExternalProject_Add_Step(hwloc_ep pre-configure - COMMAND ./autogen.sh - DEPENDEES download - DEPENDERS configure - WORKING_DIRECTORY ${HWLOC_SOURCE_DIR}) + ExternalProject_Add_Step( + hwloc_ep pre-configure + COMMAND ./autogen.sh + DEPENDEES download + DEPENDERS configure + WORKING_DIRECTORY ${HWLOC_SOURCE_DIR}) # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${HWLOC_INCLUDE_DIR}") add_library(hwloc::hwloc STATIC IMPORTED) - set_target_properties(hwloc::hwloc - PROPERTIES IMPORTED_LOCATION - "${HWLOC_LIB_DIR}/${HWLOC_STATIC_LIB_NAME}" - INTERFACE_INCLUDE_DIRECTORIES - "${HWLOC_INCLUDE_DIR}") + set_target_properties( + hwloc::hwloc + PROPERTIES IMPORTED_LOCATION "${HWLOC_LIB_DIR}/${HWLOC_STATIC_LIB_NAME}" + INTERFACE_INCLUDE_DIRECTORIES "${HWLOC_INCLUDE_DIR}") add_dependencies(hwloc::hwloc hwloc_ep) endmacro() @@ -75,36 +77,43 @@ endmacro() macro(build_memkind) message(STATUS "Building Memkind from source") set(MEMKIND_BUILD_VERSION "v1.14.0") - set(MEMKIND_BUILD_SHA256_CHECKSUM "ab366b20b5a87ea655483631fc762ba6eb59eb6c3a08652e643f1ee3f06a6a12") + set(MEMKIND_BUILD_SHA256_CHECKSUM + "ab366b20b5a87ea655483631fc762ba6eb59eb6c3a08652e643f1ee3f06a6a12") set(MEMKIND_SOURCE_URL - "https://github.com/memkind/memkind/archive/refs/tags/${MEMKIND_BUILD_VERSION}.tar.gz") + "https://github.com/memkind/memkind/archive/refs/tags/${MEMKIND_BUILD_VERSION}.tar.gz" + ) set(MEMKIND_LIB_NAME "memkind") set(MEMKIND_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/memkind_ep-install") set(MEMKIND_SOURCE_DIR "${MEMKIND_PREFIX}/src/memkind_ep") set(MEMKIND_INCLUDE_DIR "${MEMKIND_SOURCE_DIR}/include") set(MEMKIND_LIB_DIR "${MEMKIND_SOURCE_DIR}/.libs") - set(MEMKIND_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${MEMKIND_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(MEMKIND_STATIC_LIB_NAME + "${CMAKE_STATIC_LIBRARY_PREFIX}${MEMKIND_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) set(MEMKIND_STATIC_LIB_TARGETS - "${MEMKIND_SOURCE_DIR}/src/.libs/${MEMKIND_STATIC_LIB_NAME}") - set(MEMKIND_CONFIGURE_ARGS - "--prefix=${MEMKIND_PREFIX}" - "--with-pic" - "--enable-static") - ExternalProject_Add(memkind_ep - PREFIX ${MEMKIND_PREFIX} - URL ${MEMKIND_SOURCE_URL} - URL_HASH "SHA256=${MEMKIND_BUILD_SHA256_CHECKSUM}" - SOURCE_DIR ${MEMKIND_SOURCE_DIR} - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env LDFLAGS=-L${HWLOC_LIB_DIR} env CFLAGS=-I${HWLOC_INCLUDE_DIR} env CXXFLAGS=-I${HWLOC_INCLUDE_DIR} ./configure ${MEMKIND_CONFIGURE_ARGS} - BUILD_COMMAND ${MAKE} - BUILD_BYPRODUCTS ${MEMKIND_STATIC_LIB_TARGETS} - BUILD_IN_SOURCE 1) + "${MEMKIND_SOURCE_DIR}/src/.libs/${MEMKIND_STATIC_LIB_NAME}") + set(MEMKIND_CONFIGURE_ARGS "--prefix=${MEMKIND_PREFIX}" "--with-pic" + "--enable-static") + ExternalProject_Add( + memkind_ep + PREFIX ${MEMKIND_PREFIX} + URL ${MEMKIND_SOURCE_URL} + URL_HASH "SHA256=${MEMKIND_BUILD_SHA256_CHECKSUM}" + SOURCE_DIR ${MEMKIND_SOURCE_DIR} + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -E env LDFLAGS=-L${HWLOC_LIB_DIR} env + CFLAGS=-I${HWLOC_INCLUDE_DIR} env CXXFLAGS=-I${HWLOC_INCLUDE_DIR} + ./configure ${MEMKIND_CONFIGURE_ARGS} + BUILD_COMMAND ${MAKE} + BUILD_BYPRODUCTS ${MEMKIND_STATIC_LIB_TARGETS} + BUILD_IN_SOURCE 1) - ExternalProject_Add_Step(memkind_ep pre-configure - COMMAND ./autogen.sh - DEPENDEES download - DEPENDERS configure - WORKING_DIRECTORY ${MEMKIND_SOURCE_DIR}) + ExternalProject_Add_Step( + memkind_ep pre-configure + COMMAND ./autogen.sh + DEPENDEES download + DEPENDERS configure + WORKING_DIRECTORY ${MEMKIND_SOURCE_DIR}) add_dependencies(memkind_ep hwloc::hwloc) @@ -112,12 +121,12 @@ macro(build_memkind) file(MAKE_DIRECTORY "${MEMKIND_INCLUDE_DIR}") add_library(memkind::memkind STATIC IMPORTED) - set_target_properties(memkind::memkind - PROPERTIES IMPORTED_LOCATION - "${MEMKIND_LIB_DIR}/${MEMKIND_STATIC_LIB_NAME}" - INTERFACE_INCLUDE_DIRECTORIES - "${MEMKIND_INCLUDE_DIR}") - target_link_libraries(memkind::memkind INTERFACE hwloc::hwloc dl numa pthread daxctl) + set_target_properties( + memkind::memkind + PROPERTIES IMPORTED_LOCATION "${MEMKIND_LIB_DIR}/${MEMKIND_STATIC_LIB_NAME}" + INTERFACE_INCLUDE_DIRECTORIES "${MEMKIND_INCLUDE_DIR}") + target_link_libraries(memkind::memkind INTERFACE hwloc::hwloc dl numa pthread + daxctl) add_dependencies(memkind::memkind memkind_ep) endmacro() diff --git a/cpp/CMake/BuildQATZstd.cmake b/cpp/CMake/BuildQATZstd.cmake index f6c152db1460..f79e9ea58fcc 100644 --- a/cpp/CMake/BuildQATZstd.cmake +++ b/cpp/CMake/BuildQATZstd.cmake @@ -28,61 +28,72 @@ macro(build_qatzstd) include(FindZstd) message(STATUS "Building QAT-ZSTD from source") - set(QATZSTD_SOURCE_URL - "https://github.com/marin-ma/QAT-ZSTD-Plugin.git") + set(QATZSTD_SOURCE_URL "https://github.com/marin-ma/QAT-ZSTD-Plugin.git") set(QATZSTD_SOURCE_BRANCH "fix-duplicate-symbol") set(QATZSTD_LIB_NAME "qatseqprod") - set(QATZSTD_PREFIX - "${CMAKE_CURRENT_BINARY_DIR}/qatzstd_ep-install") + set(QATZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/qatzstd_ep-install") set(QATZSTD_SOURCE_DIR "${QATZSTD_PREFIX}/src/qatzstd_ep") set(QATZSTD_INCLUDE_DIR "${QATZSTD_SOURCE_DIR}/src") - set(QATZSTD_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${QATZSTD_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(QATZSTD_STATIC_LIB_TARGETS "${QATZSTD_SOURCE_DIR}/src/${QATZSTD_STATIC_LIB_NAME}") + set(QATZSTD_STATIC_LIB_NAME + "${CMAKE_STATIC_LIBRARY_PREFIX}${QATZSTD_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(QATZSTD_STATIC_LIB_TARGETS + "${QATZSTD_SOURCE_DIR}/src/${QATZSTD_STATIC_LIB_NAME}") set(QATZSTD_MAKE_ARGS "ENABLE_USDM_DRV=1" "ZSTDLIB=${ZSTD_INCLUDE_DIR}") - ExternalProject_Add(qatzstd_ep - PREFIX ${QATZSTD_PREFIX} - GIT_REPOSITORY ${QATZSTD_SOURCE_URL} - GIT_TAG ${QATZSTD_SOURCE_BRANCH} - SOURCE_DIR ${QATZSTD_SOURCE_DIR} - CONFIGURE_COMMAND "" - BUILD_COMMAND ${MAKE} ${QATZSTD_MAKE_ARGS} - INSTALL_COMMAND "" - BUILD_BYPRODUCTS ${QATZSTD_STATIC_LIB_TARGETS} - BUILD_IN_SOURCE 1) + ExternalProject_Add( + qatzstd_ep + PREFIX ${QATZSTD_PREFIX} + GIT_REPOSITORY ${QATZSTD_SOURCE_URL} + GIT_TAG ${QATZSTD_SOURCE_BRANCH} + SOURCE_DIR ${QATZSTD_SOURCE_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND ${MAKE} ${QATZSTD_MAKE_ARGS} + INSTALL_COMMAND "" + BUILD_BYPRODUCTS ${QATZSTD_STATIC_LIB_TARGETS} + BUILD_IN_SOURCE 1) add_library(qatzstd::qatzstd STATIC IMPORTED) # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${QATZSTD_INCLUDE_DIR}") - set(QATZSTD_INCLUDE_DIRS - "${QATZSTD_INCLUDE_DIR}" - "${ZSTD_INCLUDE_DIR}") + set(QATZSTD_INCLUDE_DIRS "${QATZSTD_INCLUDE_DIR}" "${ZSTD_INCLUDE_DIR}") set(QATZSTD_LINK_LIBRARIES - "${ZSTD_LIBRARY}" - "${QAT_LIBRARY}" - "${USDM_DRV_LIBRARY}" - "${ADF_LIBRARY}" + "${ZSTD_LIBRARY}" "${QAT_LIBRARY}" "${USDM_DRV_LIBRARY}" "${ADF_LIBRARY}" "${OSAL_LIBRARY}") - set_target_properties(qatzstd::qatzstd - PROPERTIES IMPORTED_LOCATION - "${QATZSTD_STATIC_LIB_TARGETS}" - INTERFACE_INCLUDE_DIRECTORIES - "${QATZSTD_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES - "${QATZSTD_LINK_LIBRARIES}") + set_target_properties( + qatzstd::qatzstd + PROPERTIES IMPORTED_LOCATION "${QATZSTD_STATIC_LIB_TARGETS}" + INTERFACE_INCLUDE_DIRECTORIES "${QATZSTD_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${QATZSTD_LINK_LIBRARIES}") add_dependencies(qatzstd::qatzstd qatzstd_ep) endmacro() -find_library(QAT_LIBRARY REQUIRED NAMES qat PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) -find_library(USDM_DRV_LIBRARY REQUIRED NAMES usdm_drv PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) -find_library(ADF_LIBRARY REQUIRED NAMES adf PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) -find_library(OSAL_LIBRARY REQUIRED NAMES osal PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) +find_library( + QAT_LIBRARY REQUIRED + NAMES qat + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) +find_library( + USDM_DRV_LIBRARY REQUIRED + NAMES usdm_drv + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) +find_library( + ADF_LIBRARY REQUIRED + NAMES adf + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) +find_library( + OSAL_LIBRARY REQUIRED + NAMES osal + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) message(STATUS "Found qat: ${QAT_LIBRARY}") message(STATUS "Found usdm_drv: ${USDM_DRV_LIBRARY}") @@ -90,4 +101,3 @@ message(STATUS "Found adf: ${ADF_LIBRARY}") message(STATUS "Found osal: ${OSAL_LIBRARY}") build_qatzstd() - diff --git a/cpp/CMake/BuildQATzip.cmake b/cpp/CMake/BuildQATzip.cmake index 376f1645509a..c68ef25ada2e 100644 --- a/cpp/CMake/BuildQATzip.cmake +++ b/cpp/CMake/BuildQATzip.cmake @@ -26,38 +26,42 @@ endif() macro(build_qatzip) message(STATUS "Building QATzip from source") set(QATZIP_BUILD_VERSION "v1.1.1") - set(QATZIP_BUILD_SHA256_CHECKSUM "679f5522deb35e7ffa36f227ae49d07ef2d69a83e56bfda849303829b274e79b") + set(QATZIP_BUILD_SHA256_CHECKSUM + "679f5522deb35e7ffa36f227ae49d07ef2d69a83e56bfda849303829b274e79b") set(QATZIP_SOURCE_URL - "https://github.com/intel/QATzip/archive/refs/tags/${QATZIP_BUILD_VERSION}.tar.gz") + "https://github.com/intel/QATzip/archive/refs/tags/${QATZIP_BUILD_VERSION}.tar.gz" + ) set(QATZIP_LIB_NAME "qatzip") - set(QATZIP_PREFIX - "${CMAKE_CURRENT_BINARY_DIR}/qatzip_ep-install") + set(QATZIP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/qatzip_ep-install") set(QATZIP_SOURCE_DIR "${QATZIP_PREFIX}/src/qatzip_ep") set(QATZIP_INCLUDE_DIR "${QATZIP_SOURCE_DIR}/include") - set(QATZIP_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${QATZIP_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(QATZIP_STATIC_LIB_NAME + "${CMAKE_STATIC_LIBRARY_PREFIX}${QATZIP_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) set(QATZIP_STATIC_LIB_TARGETS "${QATZIP_SOURCE_DIR}/src/.libs/${QATZIP_STATIC_LIB_NAME}") - set(QATZIP_CONFIGURE_ARGS - "--prefix=${QATZIP_PREFIX}" - "--with-pic" - "--with-ICP_ROOT=$ENV{ICP_ROOT}") + set(QATZIP_CONFIGURE_ARGS "--prefix=${QATZIP_PREFIX}" "--with-pic" + "--with-ICP_ROOT=$ENV{ICP_ROOT}") - ExternalProject_Add(qatzip_ep - PREFIX ${QATZIP_PREFIX} - URL ${QATZIP_SOURCE_URL} - URL_HASH "SHA256=${QATZIP_BUILD_SHA256_CHECKSUM}" - SOURCE_DIR ${QATZIP_SOURCE_DIR} - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env QZ_ROOT=${QATZIP_SOURCE_DIR} ./configure ${QATZIP_CONFIGURE_ARGS} - BUILD_COMMAND ${MAKE} all - BUILD_BYPRODUCTS ${QATZIP_STATIC_LIB_TARGETS} - BUILD_IN_SOURCE 1) + ExternalProject_Add( + qatzip_ep + PREFIX ${QATZIP_PREFIX} + URL ${QATZIP_SOURCE_URL} + URL_HASH "SHA256=${QATZIP_BUILD_SHA256_CHECKSUM}" + SOURCE_DIR ${QATZIP_SOURCE_DIR} + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env QZ_ROOT=${QATZIP_SOURCE_DIR} + ./configure ${QATZIP_CONFIGURE_ARGS} + BUILD_COMMAND ${MAKE} all + BUILD_BYPRODUCTS ${QATZIP_STATIC_LIB_TARGETS} + BUILD_IN_SOURCE 1) - ExternalProject_Add_Step(qatzip_ep pre-configure - COMMAND ./autogen.sh - DEPENDEES download - DEPENDERS configure - WORKING_DIRECTORY ${QATZIP_SOURCE_DIR}) + ExternalProject_Add_Step( + qatzip_ep pre-configure + COMMAND ./autogen.sh + DEPENDEES download + DEPENDERS configure + WORKING_DIRECTORY ${QATZIP_SOURCE_DIR}) # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${QATZIP_INCLUDE_DIR}") @@ -73,13 +77,11 @@ macro(build_qatzip) Threads::Threads) add_library(qatzip::qatzip STATIC IMPORTED) - set_target_properties(qatzip::qatzip - PROPERTIES IMPORTED_LOCATION - "${QATZIP_STATIC_LIB_TARGETS}" - INTERFACE_INCLUDE_DIRECTORIES - "${QATZIP_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES - "${QATZIP_LINK_LIBRARIES}") + set_target_properties( + qatzip::qatzip + PROPERTIES IMPORTED_LOCATION "${QATZIP_STATIC_LIB_TARGETS}" + INTERFACE_INCLUDE_DIRECTORIES "${QATZIP_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${QATZIP_LINK_LIBRARIES}") add_dependencies(qatzip::qatzip qatzip_ep) endmacro() @@ -90,10 +92,26 @@ find_package(Threads REQUIRED) find_library(ZLIB_LIBRARY REQUIRED NAMES z) find_library(LZ4_LIBRARY REQUIRED NAMES lz4) find_library(UDEV_LIBRARY REQUIRED NAMES udev) -find_library(USDM_DRV_LIBRARY REQUIRED NAMES usdm_drv PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) -find_library(QAT_LIBRARY REQUIRED NAMES qat PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) -find_library(OSAL_LIBRARY REQUIRED NAMES osal PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) -find_library(ADF_LIBRARY REQUIRED NAMES adf PATHS "$ENV{ICP_ROOT}/build" NO_DEFAULT_PATH) +find_library( + USDM_DRV_LIBRARY REQUIRED + NAMES usdm_drv + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) +find_library( + QAT_LIBRARY REQUIRED + NAMES qat + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) +find_library( + OSAL_LIBRARY REQUIRED + NAMES osal + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) +find_library( + ADF_LIBRARY REQUIRED + NAMES adf + PATHS "$ENV{ICP_ROOT}/build" + NO_DEFAULT_PATH) message(STATUS "Found zlib: ${ZLIB_LIBRARY}") message(STATUS "Found lz4: ${LZ4_LIBRARY}") @@ -102,4 +120,3 @@ message(STATUS "Found usdm_drv: ${USDM_DRV_LIBRARY}") message(STATUS "Found qat: ${QAT_LIBRARY}") build_qatzip() - diff --git a/cpp/CMake/BuildQpl.cmake b/cpp/CMake/BuildQpl.cmake index dbfd16645342..7715bb8e767f 100644 --- a/cpp/CMake/BuildQpl.cmake +++ b/cpp/CMake/BuildQpl.cmake @@ -18,46 +18,43 @@ include(ExternalProject) macro(build_qpl) - message(STATUS "Building QPL from source") - set(QPL_BUILD_VERSION "v1.1.0") - set(QPL_BUILD_SHA256_CHECKSUM "00306000035621dfbc21007481395c46ba9723fc8add8ca5142847b94dc564c5") - set(QPL_SOURCE_URL - "https://github.com/intel/qpl/archive/refs/tags/v1.1.0.tar.gz") - set(QPL_LIB_NAME "qpl") - - set(QPL_PREFIX - "${CMAKE_CURRENT_BINARY_DIR}/qpl_ep-install") - set(QPL_SOURCE_DIR "${QPL_PREFIX}/src/qpl_ep") - set(QPL_INCLUDE_DIR "${QPL_PREFIX}/include") - set(QPL_LIB_DIR "${QPL_PREFIX}/lib") - set(QPL_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${QPL_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}${QPL_STATIC_LIB_MAJOR_VERSION}") - set(QPL_STATIC_LIB_TARGETS - "${QPL_LIB_DIR}/${QPL_STATIC_LIB_NAME}" - ) - ExternalProject_Add(qpl_ep - PREFIX ${QPL_PREFIX} - URL ${QPL_SOURCE_URL} - URL_HASH "SHA256=${QPL_BUILD_SHA256_CHECKSUM}" - SOURCE_DIR ${QPL_SOURCE_DIR} - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${QPL_PREFIX} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DQPL_BUILD_TESTS=OFF - -DLOG_HW_INIT=ON - BUILD_BYPRODUCTS ${QPL_STATIC_LIB_TARGETS}) - - # The include directory must exist before it is referenced by a target. - file(MAKE_DIRECTORY "${QPL_INCLUDE_DIR}") - - add_library(qpl::qpl STATIC IMPORTED) - set_target_properties(qpl::qpl - PROPERTIES IMPORTED_LOCATION - "${QPL_LIB_DIR}/${QPL_STATIC_LIB_NAME}" - INTERFACE_INCLUDE_DIRECTORIES - "${QPL_INCLUDE_DIR}") - - add_dependencies(qpl::qpl qpl_ep) + message(STATUS "Building QPL from source") + set(QPL_BUILD_VERSION "v1.1.0") + set(QPL_BUILD_SHA256_CHECKSUM + "00306000035621dfbc21007481395c46ba9723fc8add8ca5142847b94dc564c5") + set(QPL_SOURCE_URL + "https://github.com/intel/qpl/archive/refs/tags/v1.1.0.tar.gz") + set(QPL_LIB_NAME "qpl") + + set(QPL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/qpl_ep-install") + set(QPL_SOURCE_DIR "${QPL_PREFIX}/src/qpl_ep") + set(QPL_INCLUDE_DIR "${QPL_PREFIX}/include") + set(QPL_LIB_DIR "${QPL_PREFIX}/lib") + set(QPL_STATIC_LIB_NAME + "${CMAKE_STATIC_LIBRARY_PREFIX}${QPL_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}${QPL_STATIC_LIB_MAJOR_VERSION}" + ) + set(QPL_STATIC_LIB_TARGETS "${QPL_LIB_DIR}/${QPL_STATIC_LIB_NAME}") + ExternalProject_Add( + qpl_ep + PREFIX ${QPL_PREFIX} + URL ${QPL_SOURCE_URL} + URL_HASH "SHA256=${QPL_BUILD_SHA256_CHECKSUM}" + SOURCE_DIR ${QPL_SOURCE_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${QPL_PREFIX} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DQPL_BUILD_TESTS=OFF + -DLOG_HW_INIT=ON + BUILD_BYPRODUCTS ${QPL_STATIC_LIB_TARGETS}) + + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${QPL_INCLUDE_DIR}") + + add_library(qpl::qpl STATIC IMPORTED) + set_target_properties( + qpl::qpl + PROPERTIES IMPORTED_LOCATION "${QPL_LIB_DIR}/${QPL_STATIC_LIB_NAME}" + INTERFACE_INCLUDE_DIRECTORIES "${QPL_INCLUDE_DIR}") + + add_dependencies(qpl::qpl qpl_ep) endmacro() build_qpl() - diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index 1ae4ece1b8ef..8f036be53411 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -if (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") +if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") set(ARROW_SHARED_LIBRARY_SUFFIX ".1500.dylib") set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".1500.1.0.dylib") else() @@ -30,22 +30,28 @@ set(ARROW_SUBSTRAIT_LIB_NAME "arrow_substrait") function(FIND_ARROW_LIB LIB_NAME) if(NOT TARGET Arrow::${LIB_NAME}) - set(ARROW_LIB_FULL_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}) + set(ARROW_LIB_FULL_NAME + ${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}) add_library(Arrow::${LIB_NAME} SHARED IMPORTED) - find_library(ARROW_LIB_${LIB_NAME} - NAMES ${ARROW_LIB_FULL_NAME} - PATHS ${ARROW_LIB_DIR} ${ARROW_LIB64_DIR} - NO_DEFAULT_PATH) + find_library( + ARROW_LIB_${LIB_NAME} + NAMES ${ARROW_LIB_FULL_NAME} + PATHS ${ARROW_LIB_DIR} ${ARROW_LIB64_DIR} + NO_DEFAULT_PATH) if(NOT ARROW_LIB_${LIB_NAME}) message(FATAL_ERROR "Arrow library Not Found: ${ARROW_LIB_FULL_NAME}") else() message(STATUS "Found Arrow library: ${ARROW_LIB_${LIB_NAME}}") - set_target_properties(Arrow::${LIB_NAME} + set_target_properties( + Arrow::${LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARROW_LIB_${LIB_NAME}}" - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_HOME}/install/include") + INTERFACE_INCLUDE_DIRECTORIES + "${ARROW_HOME}/install/include") endif() - file(COPY ${ARROW_LIB_${LIB_NAME}} DESTINATION ${root_directory}/releases/ FOLLOW_SYMLINK_CHAIN) + file( + COPY ${ARROW_LIB_${LIB_NAME}} + DESTINATION ${root_directory}/releases/ + FOLLOW_SYMLINK_CHAIN) endif() endfunction() diff --git a/cpp/CMake/FindThrift.cmake b/cpp/CMake/FindThrift.cmake index 07028971d9fc..273500a6ae36 100644 --- a/cpp/CMake/FindThrift.cmake +++ b/cpp/CMake/FindThrift.cmake @@ -12,27 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -# - Find Thrift (a cross platform RPC lib/tool) +# * Find Thrift (a cross platform RPC lib/tool) # # Variables used by this module, they can change the default behaviour and need # to be set before calling find_package: # -# Thrift_ROOT - When set, this path is inspected instead of standard library -# locations as the root of the Thrift installation. -# The environment variable THRIFT_HOME overrides this variable. +# Thrift_ROOT - When set, this path is inspected instead of standard library +# locations as the root of the Thrift installation. The environment variable +# THRIFT_HOME overrides this variable. # -# This module defines -# Thrift_FOUND, whether Thrift is found or not -# Thrift_COMPILER_FOUND, whether Thrift compiler is found or not +# This module defines Thrift_FOUND, whether Thrift is found or not +# Thrift_COMPILER_FOUND, whether Thrift compiler is found or not # -# thrift::thrift, a library target to use Thrift -# thrift::compiler, a executable target to use Thrift compiler +# thrift::thrift, a library target to use Thrift thrift::compiler, a executable +# target to use Thrift compiler function(EXTRACT_THRIFT_VERSION) if(THRIFT_INCLUDE_DIR) file(READ "${THRIFT_INCLUDE_DIR}/thrift/config.h" THRIFT_CONFIG_H_CONTENT) - string(REGEX MATCH "#define PACKAGE_VERSION \"[0-9.]+\"" THRIFT_VERSION_DEFINITION - "${THRIFT_CONFIG_H_CONTENT}") + string(REGEX MATCH "#define PACKAGE_VERSION \"[0-9.]+\"" + THRIFT_VERSION_DEFINITION "${THRIFT_CONFIG_H_CONTENT}") string(REGEX MATCH "[0-9.]+" Thrift_VERSION "${THRIFT_VERSION_DEFINITION}") set(Thrift_VERSION "${Thrift_VERSION}" @@ -66,14 +65,16 @@ set(THRIFT_LIB_NAME_BASE "thrift${THRIFT_MSVC_LIB_SUFFIX}") if(ARROW_THRIFT_USE_SHARED) set(THRIFT_LIB_NAMES thrift) if(CMAKE_IMPORT_LIBRARY_SUFFIX) - list(APPEND - THRIFT_LIB_NAMES - "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + list( + APPEND + THRIFT_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" ) endif() - list(APPEND - THRIFT_LIB_NAMES - "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}" + list( + APPEND + THRIFT_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}" ) else() set(THRIFT_LIB_NAMES @@ -82,20 +83,24 @@ else() endif() if(Thrift_ROOT) - find_library(THRIFT_LIB - NAMES ${THRIFT_LIB_NAMES} - PATHS ${Thrift_ROOT} - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") - find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h - PATHS ${Thrift_ROOT} - PATH_SUFFIXES "include") - find_program(THRIFT_COMPILER thrift - PATHS ${Thrift_ROOT} - PATH_SUFFIXES "bin") + find_library( + THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} + PATHS ${Thrift_ROOT} + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + find_path( + THRIFT_INCLUDE_DIR thrift/Thrift.h + PATHS ${Thrift_ROOT} + PATH_SUFFIXES "include") + find_program( + THRIFT_COMPILER thrift + PATHS ${Thrift_ROOT} + PATH_SUFFIXES "bin") extract_thrift_version() else() - # THRIFT-4760: The pkgconfig files are currently only installed when using autotools. - # Starting with 0.13, they are also installed for the CMake-based installations of Thrift. + # THRIFT-4760: The pkgconfig files are currently only installed when using + # autotools. Starting with 0.13, they are also installed for the CMake-based + # installations of Thrift. find_package(PkgConfig QUIET) pkg_check_modules(THRIFT_PC thrift) if(THRIFT_PC_FOUND) @@ -103,19 +108,22 @@ else() list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") - find_library(THRIFT_LIB - NAMES ${THRIFT_LIB_NAMES} - PATHS ${THRIFT_PC_LIBRARY_DIRS} - NO_DEFAULT_PATH) - find_program(THRIFT_COMPILER thrift - HINTS ${THRIFT_PC_PREFIX} - NO_DEFAULT_PATH - PATH_SUFFIXES "bin") + find_library( + THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} + PATHS ${THRIFT_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) + find_program( + THRIFT_COMPILER thrift + HINTS ${THRIFT_PC_PREFIX} + NO_DEFAULT_PATH + PATH_SUFFIXES "bin") set(Thrift_VERSION ${THRIFT_PC_VERSION}) else() - find_library(THRIFT_LIB - NAMES ${THRIFT_LIB_NAMES} - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + find_library( + THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include") find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin") extract_thrift_version() @@ -140,14 +148,15 @@ if(Thrift_FOUND) else() add_library(thrift::thrift STATIC IMPORTED) endif() - set_target_properties(thrift::thrift - PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") + set_target_properties( + thrift::thrift + PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" INTERFACE_INCLUDE_DIRECTORIES + "${THRIFT_INCLUDE_DIR}") if(WIN32 AND NOT MSVC_TOOLCHAIN) - # We don't need this for Visual C++ because Thrift uses - # "#pragma comment(lib, "Ws2_32.lib")" in - # thrift/windows/config.h for Visual C++. - set_target_properties(thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES "ws2_32") + # We don't need this for Visual C++ because Thrift uses "#pragma + # comment(lib, "Ws2_32.lib")" in thrift/windows/config.h for Visual C++. + set_target_properties(thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES + "ws2_32") endif() if(Thrift_COMPILER_FOUND) diff --git a/cpp/CMake/FindZstd.cmake b/cpp/CMake/FindZstd.cmake index a7efd5adfd3f..62e8b874d735 100644 --- a/cpp/CMake/FindZstd.cmake +++ b/cpp/CMake/FindZstd.cmake @@ -15,43 +15,44 @@ # specific language governing permissions and limitations # under the License. -# ZSTD_HOME environmental variable is used to check for Zstd headers and static library +# ZSTD_HOME environmental variable is used to check for Zstd headers and static +# library -# ZSTD_INCLUDE_DIR: directory containing headers -# ZSTD_LIBRARY: path to libzstd.so -# ZSTD_FOUND: whether zstd has been found +# ZSTD_INCLUDE_DIR: directory containing headers ZSTD_LIBRARY: path to +# libzstd.so ZSTD_FOUND: whether zstd has been found -if (NOT "$ENV{ZSTD_HOME}" STREQUAL "") +if(NOT "$ENV{ZSTD_HOME}" STREQUAL "") file(TO_CMAKE_PATH "$ENV{ZSTD_HOME}" _zstd_path) message(STATUS "ZSTD_HOME: ${_zstd_path}") else() set(_zstd_path "/usr/local") endif() -find_path(ZSTD_INCLUDE_DIR zstd.h HINTS - ${_zstd_path} - PATH_SUFFIXES "include") +find_path( + ZSTD_INCLUDE_DIR zstd.h + HINTS ${_zstd_path} + PATH_SUFFIXES "include") -find_library (ZSTD_LIBRARY NAMES zstd HINTS - ${_zstd_path} - PATH_SUFFIXES "lib") +find_library( + ZSTD_LIBRARY + NAMES zstd + HINTS ${_zstd_path} + PATH_SUFFIXES "lib") -if (ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY) +if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY) set(ZSTD_FOUND TRUE) set(ZSTD_HEADER_NAME zstd.h) set(ZSTD_HEADER ${ZSTD_INCLUDE_DIR}/${ZSTD_HEADER_NAME}) -else () +else() set(ZSTD_FOUND FALSE) -endif () +endif() -if (ZSTD_FOUND) +if(ZSTD_FOUND) message(STATUS "Found the zstd header: ${ZSTD_HEADER}") message(STATUS "Found the zstd static library: ${ZSTD_LIBRARY}") -else () - message(FATAL_ERROR ZSTD_ERR_MSG "Could not find zstd. Looked in ${_zstd_path}.") -endif () - -mark_as_advanced( - ZSTD_INCLUDE_DIR - ZSTD_LIBRARY) +else() + message(FATAL_ERROR ZSTD_ERR_MSG + "Could not find zstd. Looked in ${_zstd_path}.") +endif() +mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) diff --git a/cpp/CMake/Findglog.cmake b/cpp/CMake/Findglog.cmake index b165fd80f3d7..6d9dbdacf1b1 100644 --- a/cpp/CMake/Findglog.cmake +++ b/cpp/CMake/Findglog.cmake @@ -22,23 +22,17 @@ if(NOT BUILD_GLOG) include(FindPackageHandleStandardArgs) include(SelectLibraryConfigurations) - find_library(GLOG_LIBRARY_RELEASE glog - PATHS ${GLOG_LIBRARYDIR}) - find_library(GLOG_LIBRARY_DEBUG glogd - PATHS ${GLOG_LIBRARYDIR}) + find_library(GLOG_LIBRARY_RELEASE glog PATHS ${GLOG_LIBRARYDIR}) + find_library(GLOG_LIBRARY_DEBUG glogd PATHS ${GLOG_LIBRARYDIR}) - find_path(GLOG_INCLUDE_DIR glog/logging.h - PATHS ${GLOG_INCLUDEDIR}) + find_path(GLOG_INCLUDE_DIR glog/logging.h PATHS ${GLOG_INCLUDEDIR}) select_library_configurations(GLOG) - find_package_handle_standard_args(glog DEFAULT_MSG - GLOG_LIBRARY - GLOG_INCLUDE_DIR) + find_package_handle_standard_args(glog DEFAULT_MSG GLOG_LIBRARY + GLOG_INCLUDE_DIR) - mark_as_advanced( - GLOG_LIBRARY - GLOG_INCLUDE_DIR) + mark_as_advanced(GLOG_LIBRARY GLOG_INCLUDE_DIR) endif() if(NOT glog_FOUND) @@ -56,26 +50,40 @@ endif() # glog::glog may already exist. Use google::glog to avoid conflicts. add_library(google::glog ${libglog_type} IMPORTED) -set_target_properties(google::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIR}") -set_target_properties(google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARY}") +set_target_properties(google::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${GLOG_INCLUDE_DIR}") +set_target_properties( + google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${GLOG_LIBRARY}") set(GLUTEN_GFLAGS_VERSION 2.2.2) -find_package(gflags ${GLUTEN_GFLAGS_VERSION} CONFIG COMPONENTS ${libgflags_component}) +find_package(gflags ${GLUTEN_GFLAGS_VERSION} CONFIG + COMPONENTS ${libgflags_component}) if(NOT gflags_FOUND AND glog_FOUND) - message(FATAL_ERROR "Glog found but Gflags not found. Set BUILD_GLOG=ON and reload cmake.") + message( + FATAL_ERROR + "Glog found but Gflags not found. Set BUILD_GLOG=ON and reload cmake.") endif() if(gflags_FOUND) - if(NOT TARGET gflags::gflags_${libgflags_component} AND NOT TARGET gflags_${libgflags_component}) - message(FATAL_ERROR "Found Gflags but missing component gflags_${libgflags_component}") + if(NOT TARGET gflags::gflags_${libgflags_component} + AND NOT TARGET gflags_${libgflags_component}) + message( + FATAL_ERROR + "Found Gflags but missing component gflags_${libgflags_component}") endif() if(TARGET gflags::gflags_${libgflags_component}) - set_target_properties(google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES gflags::gflags_${libgflags_component}) + set_target_properties( + google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES + gflags::gflags_${libgflags_component}) else() - set_target_properties(google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES gflags_${libgflags_component}) + set_target_properties( + google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES + gflags_${libgflags_component}) endif() else() include(BuildGflags) - set_target_properties(google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES gflags_static) + set_target_properties( + google::glog PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES gflags_static) endif() diff --git a/cpp/CMake/Findjemalloc_pic.cmake b/cpp/CMake/Findjemalloc_pic.cmake index 9511dcd33663..fae9f0d7ad80 100644 --- a/cpp/CMake/Findjemalloc_pic.cmake +++ b/cpp/CMake/Findjemalloc_pic.cmake @@ -20,7 +20,7 @@ macro(find_jemalloc) # Find the existing Protobuf set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_package(jemalloc_pic) - if ("${Jemalloc_LIBRARY}" STREQUAL "Jemalloc_LIBRARY-NOTFOUND") + if("${Jemalloc_LIBRARY}" STREQUAL "Jemalloc_LIBRARY-NOTFOUND") message(FATAL_ERROR "Jemalloc Library Not Found") endif() set(PROTOC_BIN ${Jemalloc_PROTOC_EXECUTABLE}) @@ -35,22 +35,18 @@ macro(build_jemalloc) else() set(JEMALLOC_BUILD_VERSION "5.2.1") set(JEMALLOC_SOURCE_URL - "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_BUILD_VERSION}/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" - "https://github.com/ursa-labs/thirdparty/releases/download/latest/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" - ) + "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_BUILD_VERSION}/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" + "https://github.com/ursa-labs/thirdparty/releases/download/latest/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" + ) endif() set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-install") set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib") set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include") - set( - JEMALLOC_STATIC_LIB - "${JEMALLOC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set( - JEMALLOC_INCLUDE - "${JEMALLOC_PREFIX}/include" - ) + set(JEMALLOC_STATIC_LIB + "${JEMALLOC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(JEMALLOC_INCLUDE "${JEMALLOC_PREFIX}/include") set(JEMALLOC_CONFIGURE_ARGS "AR=${CMAKE_AR}" "CC=${CMAKE_C_COMPILER}" @@ -66,14 +62,15 @@ macro(build_jemalloc) "CFLAGS=-fPIC" "CXXFLAGS=-fPIC") set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) - ExternalProject_Add(jemalloc_ep - URL ${JEMALLOC_SOURCE_URL} - PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html - CONFIGURE_COMMAND "./configure" ${JEMALLOC_CONFIGURE_ARGS} - BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND} - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" - INSTALL_COMMAND make install) + ExternalProject_Add( + jemalloc_ep + URL ${JEMALLOC_SOURCE_URL} + PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html + CONFIGURE_COMMAND "./configure" ${JEMALLOC_CONFIGURE_ARGS} + BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" + INSTALL_COMMAND make install) file(MAKE_DIRECTORY "${JEMALLOC_INCLUDE_DIR}") add_library(jemalloc::libjemalloc STATIC IMPORTED) @@ -81,7 +78,6 @@ macro(build_jemalloc) jemalloc::libjemalloc PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES - "${JEMALLOC_INCLUDE_DIR}") + INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}") add_dependencies(jemalloc::libjemalloc protobuf_ep) endmacro() diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d6e3eeb133f7..3ee336dd6a14 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -19,13 +19,13 @@ message(STATUS "Building using CMake version: ${CMAKE_VERSION}") set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# The set(CACHE) command does not remove any normal variable of the same name from the current scope -# https://cmake.org/cmake/help/latest/policy/CMP0126.html +# The set(CACHE) command does not remove any normal variable of the same name +# from the current scope https://cmake.org/cmake/help/latest/policy/CMP0126.html if(POLICY CMP0126) cmake_policy(SET CMP0126 NEW) endif() -if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() @@ -36,7 +36,9 @@ if(NOT DEFINED CMAKE_BUILD_TYPE) endif() set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "") +set(CMAKE_EXPORT_COMPILE_COMMANDS + ON + CACHE INTERNAL "") project(gluten) @@ -60,12 +62,12 @@ option(ENABLE_ABFS "Enable ABFS" OFF) set(root_directory ${PROJECT_BINARY_DIR}) get_filename_component(GLUTEN_HOME ${CMAKE_SOURCE_DIR} DIRECTORY) -if (NOT DEFINED VELOX_HOME) +if(NOT DEFINED VELOX_HOME) set(VELOX_HOME ${GLUTEN_HOME}/ep/build-velox/build/velox_ep) message(STATUS "Set VELOX_HOME to ${VELOX_HOME}") endif() -if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") +if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") set(ARROW_HOME ${VELOX_HOME}/_build/debug/third_party/arrow_ep) else() set(ARROW_HOME ${VELOX_HOME}/_build/release/third_party/arrow_ep) @@ -77,10 +79,10 @@ include(ResolveDependency) # Compiler flags # -if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") +if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb -O0") message(STATUS "CMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}") -else () +else() add_definitions(-DNDEBUG) message(STATUS "Add definition NDEBUG") endif() @@ -97,22 +99,20 @@ set(KNOWN_WARNINGS -Wno-ignored-qualifiers") if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - set(KNOWN_WARNINGS - "-Wno-error=unused-but-set-variable \ + set(KNOWN_WARNINGS "-Wno-error=unused-but-set-variable \ ${KNOWN_WARNINGS}") if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11) - set(KNOWN_WARNINGS - "-Wno-error=maybe-uninitialized \ + set(KNOWN_WARNINGS "-Wno-error=maybe-uninitialized \ ${KNOWN_WARNINGS}") endif() -elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") +elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") # Experimental set(KNOWN_WARNINGS "-Wno-implicit-int-float-conversion \ -Wno-nullability-completeness \ -Wno-mismatched-tags \ ${KNOWN_WARNINGS}") -elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") +elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") # Experimental set(KNOWN_WARNINGS "-Wno-implicit-int-float-conversion \ @@ -126,15 +126,16 @@ else() endif() # see https://issues.apache.org/jira/browse/ARROW-4665 -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(KNOWN_WARNINGS "-Wno-macro-redefined \ -Wno-nullability-completeness \ -Wno-pessimizing-move \ -Wno-mismatched-tags \ ${KNOWN_WARNINGS}") - # Specific definition for an issue with boost/stacktrace when building on macOS. - # See https://github.com/boostorg/stacktrace/issues/88 and comments therein. + # Specific definition for an issue with boost/stacktrace when building on + # macOS. See https://github.com/boostorg/stacktrace/issues/88 and comments + # therein. add_compile_definitions(_GNU_SOURCE) endif() @@ -166,17 +167,11 @@ endif() function(ADD_TEST_CASE TEST_NAME) set(options) set(one_value_args) - set(multi_value_args - SOURCES - EXTRA_LINK_LIBS - EXTRA_INCLUDES - EXTRA_DEPENDENCIES) - - cmake_parse_arguments(ARG - "${options}" - "${one_value_args}" - "${multi_value_args}" - ${ARGN}) + set(multi_value_args SOURCES EXTRA_LINK_LIBS EXTRA_INCLUDES + EXTRA_DEPENDENCIES) + + cmake_parse_arguments(ARG "${options}" "${one_value_args}" + "${multi_value_args}" ${ARGN}) if(ARG_SOURCES) set(SOURCES ${ARG_SOURCES}) @@ -185,7 +180,8 @@ function(ADD_TEST_CASE TEST_NAME) endif() add_executable(${TEST_NAME} ${SOURCES}) - target_link_libraries(${TEST_NAME} gluten google::glog GTest::gtest GTest::gtest_main Threads::Threads) + target_link_libraries(${TEST_NAME} gluten google::glog GTest::gtest + GTest::gtest_main Threads::Threads) target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/core) if(ARG_EXTRA_LINK_LIBS) diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index dc9ce3435c38..3a4d6e9e8792 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -23,25 +23,27 @@ include(GNUInstallDirs) include(CheckCXXCompilerFlag) # Only set arch=native for non-AppleClang compilers. -if (NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") +if(NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") endif() set(BOOST_MIN_VERSION "1.42.0") find_package(Boost REQUIRED) -INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}) +include_directories(${Boost_INCLUDE_DIRS}) set(source_root_directory ${CMAKE_CURRENT_SOURCE_DIR}) -if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) -set(SUBSTRAIT_PROTO_SRC_DIR ${GLUTEN_HOME}/gluten-core/src/main/resources/substrait/proto) +set(SUBSTRAIT_PROTO_SRC_DIR + ${GLUTEN_HOME}/gluten-core/src/main/resources/substrait/proto) message(STATUS "Set Substrait Proto Directory in ${SUBSTRAIT_PROTO_SRC_DIR}") -set(GLUTEN_PROTO_SRC_DIR ${GLUTEN_HOME}/gluten-core/src/main/resources/org/apache/gluten/proto) +set(GLUTEN_PROTO_SRC_DIR + ${GLUTEN_HOME}/gluten-core/src/main/resources/org/apache/gluten/proto) message(STATUS "Set Gluten Proto Directory in ${GLUTEN_PROTO_SRC_DIR}") find_program(CCACHE_FOUND ccache) @@ -58,31 +60,25 @@ macro(build_protobuf) set(PROTOBUF_SOURCE_URL "$ENV{GLUTEN_PROTOBUF_URL}") else() set(PROTOBUF_BUILD_VERSION "21.4") - set (PROTOBUF_SOURCE_URL - "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_BUILD_VERSION}/protobuf-all-${PROTOBUF_BUILD_VERSION}.tar.gz") + set(PROTOBUF_SOURCE_URL + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_BUILD_VERSION}/protobuf-all-${PROTOBUF_BUILD_VERSION}.tar.gz" + ) endif() - set(PROTOBUF_BUILD_SHA256_CHECKSUM "6c5e1b0788afba4569aeebb2cfe205cb154aa01deacaba0cd26442f3b761a836") + set(PROTOBUF_BUILD_SHA256_CHECKSUM + "6c5e1b0788afba4569aeebb2cfe205cb154aa01deacaba0cd26442f3b761a836") set(PROTOBUF_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/protobuf_ep-install") set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") - set( - PROTOBUF_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set( - PROTOC_STATIC_LIB - "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set( - PROTOC_BIN - "${PROTOBUF_PREFIX}/bin/protoc" - ) - set( - PROTOBUF_INCLUDE - "${PROTOBUF_PREFIX}/include" - CACHE PATH - "Protobuf include path" - ) + set(PROTOBUF_STATIC_LIB + "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(PROTOC_STATIC_LIB + "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(PROTOC_BIN "${PROTOBUF_PREFIX}/bin/protoc") + set(PROTOBUF_INCLUDE + "${PROTOBUF_PREFIX}/include" + CACHE PATH "Protobuf include path") set(PROTOBUF_COMPILER "${PROTOBUF_PREFIX}/bin/protoc") set(PROTOBUF_CONFIGURE_ARGS "AR=${CMAKE_AR}" @@ -94,22 +90,23 @@ macro(build_protobuf) "CFLAGS=-fPIC" "CXXFLAGS=-fPIC") set(PROTOBUF_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) - ExternalProject_Add(protobuf_ep - PREFIX protobuf_ep - CONFIGURE_COMMAND ./autogen.sh COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS} - BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_COMPILER}" - BUILD_COMMAND ${PROTOBUF_BUILD_COMMAND} - BUILD_IN_SOURCE 1 - URL ${PROTOBUF_SOURCE_URL} - URL_HASH "SHA256=${PROTOBUF_BUILD_SHA256_CHECKSUM}" - ) + ExternalProject_Add( + protobuf_ep + PREFIX protobuf_ep + CONFIGURE_COMMAND ./autogen.sh + COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS} + BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_COMPILER}" + BUILD_COMMAND ${PROTOBUF_BUILD_COMMAND} + BUILD_IN_SOURCE 1 + URL ${PROTOBUF_SOURCE_URL} + URL_HASH "SHA256=${PROTOBUF_BUILD_SHA256_CHECKSUM}") file(MAKE_DIRECTORY "${PROTOBUF_INCLUDE_DIR}") add_library(protobuf::libprotobuf STATIC IMPORTED) set_target_properties( protobuf::libprotobuf - PROPERTIES IMPORTED_LOCATION "${PROTOBUF_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES - "${PROTOBUF_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${PROTOBUF_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${PROTOBUF_INCLUDE_DIR}") add_dependencies(protobuf::libprotobuf protobuf_ep) endmacro() @@ -117,17 +114,19 @@ macro(find_protobuf) # Find the existing Protobuf set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_package(Protobuf) - if ("${Protobuf_LIBRARY}" STREQUAL "Protobuf_LIBRARY-NOTFOUND") + if("${Protobuf_LIBRARY}" STREQUAL "Protobuf_LIBRARY-NOTFOUND") message(FATAL_ERROR "Protobuf Library Not Found") endif() set(PROTOC_BIN ${Protobuf_PROTOC_EXECUTABLE}) - set(PROTOBUF_INCLUDE "${Protobuf_INCLUDE_DIRS}" CACHE PATH "Protobuf include path") + set(PROTOBUF_INCLUDE + "${Protobuf_INCLUDE_DIRS}" + CACHE PATH "Protobuf include path") endmacro() if(USE_AVX512) # Only enable additional instruction sets if they are supported message(STATUS "System processor: ${CMAKE_SYSTEM_PROCESSOR}") - if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") set(AVX512_FLAG "-march=skylake-avx512") check_cxx_compiler_flag(${AVX512_FLAG} CXX_SUPPORTS_AVX512) if(NOT CXX_SUPPORTS_AVX512) @@ -135,7 +134,7 @@ if(USE_AVX512) endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX512_FLAG}") add_definitions(-DCOLUMNAR_PLUGIN_USE_AVX512) - endif () + endif() endif() # Set up Proto @@ -144,83 +143,90 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/proto) # List Substrait Proto compiled files file(GLOB SUBSTRAIT_PROTO_FILES ${SUBSTRAIT_PROTO_SRC_DIR}/substrait/*.proto - ${SUBSTRAIT_PROTO_SRC_DIR}/substrait/extensions/*.proto) -FOREACH(PROTO ${SUBSTRAIT_PROTO_FILES}) + ${SUBSTRAIT_PROTO_SRC_DIR}/substrait/extensions/*.proto) +foreach(PROTO ${SUBSTRAIT_PROTO_FILES}) file(RELATIVE_PATH REL_PROTO ${SUBSTRAIT_PROTO_SRC_DIR} ${PROTO}) string(REGEX REPLACE "\\.proto" "" PROTO_NAME ${REL_PROTO}) - LIST(APPEND SUBSTRAIT_PROTO_SRCS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.cc") - LIST(APPEND SUBSTRAIT_PROTO_HDRS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.h") -ENDFOREACH() -set(SUBSTRAIT_PROTO_OUTPUT_FILES ${SUBSTRAIT_PROTO_HDRS} ${SUBSTRAIT_PROTO_SRCS}) -set_source_files_properties(${SUBSTRAIT_PROTO_OUTPUT_FILES} PROPERTIES GENERATED TRUE) -get_filename_component(SUBSTRAIT_PROTO_DIR ${SUBSTRAIT_PROTO_SRC_DIR}/ DIRECTORY) + list(APPEND SUBSTRAIT_PROTO_SRCS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.cc") + list(APPEND SUBSTRAIT_PROTO_HDRS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.h") +endforeach() +set(SUBSTRAIT_PROTO_OUTPUT_FILES ${SUBSTRAIT_PROTO_HDRS} + ${SUBSTRAIT_PROTO_SRCS}) +set_source_files_properties(${SUBSTRAIT_PROTO_OUTPUT_FILES} PROPERTIES GENERATED + TRUE) +get_filename_component(SUBSTRAIT_PROTO_DIR ${SUBSTRAIT_PROTO_SRC_DIR}/ + DIRECTORY) # List Gluten Proto compiled files file(GLOB GLUTEN_PROTO_FILES ${GLUTEN_PROTO_SRC_DIR}/*.proto) -FOREACH(PROTO ${GLUTEN_PROTO_FILES}) +foreach(PROTO ${GLUTEN_PROTO_FILES}) file(RELATIVE_PATH REL_PROTO ${GLUTEN_PROTO_SRC_DIR} ${PROTO}) string(REGEX REPLACE "\\.proto" "" PROTO_NAME ${REL_PROTO}) - LIST(APPEND GLUTEN_PROTO_SRCS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.cc") - LIST(APPEND GLUTEN_PROTO_HDRS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.h") -ENDFOREACH() + list(APPEND GLUTEN_PROTO_SRCS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.cc") + list(APPEND GLUTEN_PROTO_HDRS "${PROTO_OUTPUT_DIR}/${PROTO_NAME}.pb.h") +endforeach() set(GLUTEN_PROTO_OUTPUT_FILES ${GLUTEN_PROTO_HDRS} ${GLUTEN_PROTO_SRCS}) -set_source_files_properties(${GLUTEN_PROTO_OUTPUT_FILES} PROPERTIES GENERATED TRUE) +set_source_files_properties(${GLUTEN_PROTO_OUTPUT_FILES} PROPERTIES GENERATED + TRUE) get_filename_component(GLUTEN_PROTO_DIR ${GLUTEN_PROTO_SRC_DIR}/ DIRECTORY) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes") +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes") message("Core module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") set(SPARK_COLUMNAR_PLUGIN_SRCS - ${SUBSTRAIT_PROTO_SRCS} - ${GLUTEN_PROTO_SRCS} - compute/Runtime.cc - compute/ProtobufUtils.cc - compute/ResultIterator.cc - config/GlutenConfig.cc - jni/JniWrapper.cc - memory/AllocationListener.cc - memory/MemoryAllocator.cc - memory/ArrowMemoryPool.cc - memory/ColumnarBatch.cc - operators/writer/ArrowWriter.cc - shuffle/FallbackRangePartitioner.cc - shuffle/HashPartitioner.cc - shuffle/LocalPartitionWriter.cc - shuffle/Options.cc - shuffle/Partitioner.cc - shuffle/Partitioning.cc - shuffle/Payload.cc - shuffle/rss/RssPartitionWriter.cc - shuffle/RoundRobinPartitioner.cc - shuffle/ShuffleMemoryPool.cc - shuffle/ShuffleReader.cc - shuffle/SinglePartitioner.cc - shuffle/Spill.cc - shuffle/Utils.cc - utils/Compression.cc - utils/StringUtil.cc - utils/ObjectStore.cc - jni/JniError.cc - jni/JniCommon.cc) + ${SUBSTRAIT_PROTO_SRCS} + ${GLUTEN_PROTO_SRCS} + compute/Runtime.cc + compute/ProtobufUtils.cc + compute/ResultIterator.cc + config/GlutenConfig.cc + jni/JniWrapper.cc + memory/AllocationListener.cc + memory/MemoryAllocator.cc + memory/ArrowMemoryPool.cc + memory/ColumnarBatch.cc + operators/writer/ArrowWriter.cc + shuffle/FallbackRangePartitioner.cc + shuffle/HashPartitioner.cc + shuffle/LocalPartitionWriter.cc + shuffle/Options.cc + shuffle/Partitioner.cc + shuffle/Partitioning.cc + shuffle/Payload.cc + shuffle/rss/RssPartitionWriter.cc + shuffle/RoundRobinPartitioner.cc + shuffle/ShuffleMemoryPool.cc + shuffle/ShuffleReader.cc + shuffle/SinglePartitioner.cc + shuffle/Spill.cc + shuffle/Utils.cc + utils/Compression.cc + utils/StringUtil.cc + utils/ObjectStore.cc + jni/JniError.cc + jni/JniCommon.cc) file(MAKE_DIRECTORY ${root_directory}/releases) add_library(gluten SHARED ${SPARK_COLUMNAR_PLUGIN_SRCS}) add_dependencies(gluten jni_proto) if(ENABLE_GLUTEN_VCPKG) - # Hide symbols of some static dependencies. Otherwise, if such dependencies are already - # statically linked to libvelox.so, a runtime error will be reported: xxx is being linked - # both statically and dynamically. - target_link_options(gluten PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) + # Hide symbols of some static dependencies. Otherwise, if such dependencies + # are already statically linked to libvelox.so, a runtime error will be + # reported: xxx is being linked both statically and dynamically. + target_link_options( + gluten PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) endif() if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) - execute_process(COMMAND ${CMAKE_C_COMPILER} -print-file-name=libstdc++fs.a + execute_process( + COMMAND ${CMAKE_C_COMPILER} -print-file-name=libstdc++fs.a RESULT_VARIABLE LIBSTDCXXFS_STATIC_RESULT OUTPUT_VARIABLE LIBSTDCXXFS_STATIC_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) - if (LIBSTDCXXFS_STATIC_RESULT EQUAL 0 AND EXISTS "${LIBSTDCXXFS_STATIC_PATH}") + if(LIBSTDCXXFS_STATIC_RESULT EQUAL 0 AND EXISTS "${LIBSTDCXXFS_STATIC_PATH}") message(STATUS "libstdc++fs.a found at: ${LIBSTDCXXFS_STATIC_PATH}") target_link_libraries(gluten PRIVATE ${LIBSTDCXXFS_STATIC_PATH}) else() @@ -243,57 +249,55 @@ if(ENABLE_QAT) include(BuildQATzip) include(BuildQATZstd) target_sources(gluten PRIVATE utils/qat/QatCodec.cc) - target_include_directories(gluten PUBLIC ${QATZIP_INCLUDE_DIR} ${QATZSTD_INCLUDE_DIR}) + target_include_directories(gluten PUBLIC ${QATZIP_INCLUDE_DIR} + ${QATZSTD_INCLUDE_DIR}) target_link_libraries(gluten PUBLIC qatzip::qatzip qatzstd::qatzstd) endif() if(ENABLE_IAA) include(BuildQpl) target_include_directories(gluten PUBLIC ${QPL_INCLUDE_DIR}) - target_sources(gluten PRIVATE utils/qpl/qpl_job_pool.cc utils/qpl/qpl_codec.cc) + target_sources(gluten PRIVATE utils/qpl/qpl_job_pool.cc + utils/qpl/qpl_codec.cc) target_link_libraries(gluten PUBLIC qpl::qpl) endif() if(BUILD_PROTOBUF) build_protobuf() message(STATUS "Building ProtoBuf from Source: ${BUILD_PROTOBUF}") - target_link_libraries(gluten - LINK_PRIVATE protobuf::libprotobuf) + target_link_libraries(gluten LINK_PRIVATE protobuf::libprotobuf) else() find_protobuf() message(STATUS "Use existing ProtoBuf libraries: ${PROTOBUF_LIBRARY}") - target_link_libraries(gluten - LINK_PUBLIC ${PROTOBUF_LIBRARY}) + target_link_libraries(gluten LINK_PUBLIC ${PROTOBUF_LIBRARY}) endif() -add_custom_command(OUTPUT ${SUBSTRAIT_PROTO_OUTPUT_FILES} - COMMAND ${PROTOC_BIN} - --proto_path - ${SUBSTRAIT_PROTO_SRC_DIR}/ - --cpp_out - ${PROTO_OUTPUT_DIR} - ${SUBSTRAIT_PROTO_FILES} - DEPENDS ${SUBSTRAIT_PROTO_DIR} - COMMENT "Running Substrait PROTO compiler" - VERBATIM) - -add_custom_command(OUTPUT ${GLUTEN_PROTO_OUTPUT_FILES} - COMMAND ${PROTOC_BIN} - --proto_path - ${GLUTEN_PROTO_SRC_DIR}/ - --cpp_out - ${PROTO_OUTPUT_DIR} - ${GLUTEN_PROTO_FILES} - DEPENDS ${GLUTEN_PROTO_DIR} - COMMENT "Running Gluten PROTO compiler" - VERBATIM) - -add_custom_target(jni_proto ALL DEPENDS ${SUBSTRAIT_PROTO_OUTPUT_FILES} ${GLUTEN_PROTO_OUTPUT_FILES}) +add_custom_command( + OUTPUT ${SUBSTRAIT_PROTO_OUTPUT_FILES} + COMMAND ${PROTOC_BIN} --proto_path ${SUBSTRAIT_PROTO_SRC_DIR}/ --cpp_out + ${PROTO_OUTPUT_DIR} ${SUBSTRAIT_PROTO_FILES} + DEPENDS ${SUBSTRAIT_PROTO_DIR} + COMMENT "Running Substrait PROTO compiler" + VERBATIM) + +add_custom_command( + OUTPUT ${GLUTEN_PROTO_OUTPUT_FILES} + COMMAND ${PROTOC_BIN} --proto_path ${GLUTEN_PROTO_SRC_DIR}/ --cpp_out + ${PROTO_OUTPUT_DIR} ${GLUTEN_PROTO_FILES} + DEPENDS ${GLUTEN_PROTO_DIR} + COMMENT "Running Gluten PROTO compiler" + VERBATIM) + +add_custom_target(jni_proto ALL DEPENDS ${SUBSTRAIT_PROTO_OUTPUT_FILES} + ${GLUTEN_PROTO_OUTPUT_FILES}) add_dependencies(jni_proto protobuf::libprotobuf) -target_include_directories(gluten PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} ${JNI_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR} ${PROTO_OUTPUT_DIR} ${PROTOBUF_INCLUDE}) -set_target_properties(gluten PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${root_directory}/releases) +target_include_directories( + gluten + PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} ${JNI_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR} ${PROTO_OUTPUT_DIR} ${PROTOBUF_INCLUDE}) +set_target_properties(gluten PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ${root_directory}/releases) include(Findjemalloc_pic) # Build Jemalloc @@ -313,23 +317,26 @@ if(BUILD_BENCHMARKS) add_subdirectory(benchmarks) endif() - if(DEFINED ENV{HADOOP_HOME}) set(LIBHDFS3_DESTINATION $ENV{HADOOP_HOME}/lib/native) else() set(LIBHDFS3_DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() -target_link_libraries(gluten - PUBLIC Arrow::arrow Arrow::parquet) +target_link_libraries(gluten PUBLIC Arrow::arrow Arrow::parquet) target_link_libraries(gluten PRIVATE google::glog) -install(TARGETS gluten - DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR}) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/resources/libhdfs.so DESTINATION ${LIBHDFS3_DESTINATION}) -add_custom_command(TARGET gluten POST_BUILD COMMAND ld $ || true - COMMENT "Checking ld result of libgluten.so") -add_custom_command(TARGET gluten POST_BUILD COMMAND ldd $ || true - COMMENT "Checking ldd result of libgluten.so") +add_custom_command( + TARGET gluten + POST_BUILD + COMMAND ld $ || true + COMMENT "Checking ld result of libgluten.so") +add_custom_command( + TARGET gluten + POST_BUILD + COMMAND ldd $ || true + COMMENT "Checking ldd result of libgluten.so") diff --git a/cpp/core/benchmarks/CMakeLists.txt b/cpp/core/benchmarks/CMakeLists.txt index 6d39501477df..4b4c7656639c 100644 --- a/cpp/core/benchmarks/CMakeLists.txt +++ b/cpp/core/benchmarks/CMakeLists.txt @@ -31,7 +31,8 @@ macro(package_add_gbenchmark TESTNAME) add_executable(${TESTNAME} ${ARGN}) - target_link_libraries(${TESTNAME} benchmark::benchmark gluten google::glog ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${TESTNAME} benchmark::benchmark gluten google::glog + ${CMAKE_THREAD_LIBS_INIT}) target_include_directories(${TESTNAME} PUBLIC ${source_root_directory}) set_target_properties(${TESTNAME} PROPERTIES FOLDER tests) endmacro() diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 34cc9001cf38..6d66ea506a7e 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -23,39 +23,54 @@ include(GNUInstallDirs) include(CheckCXXCompilerFlag) include(FindPackageHandleStandardArgs) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes") -if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations -Wno-attributes") +if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess") endif() -set(SYSTEM_LIB_PATH "/usr/lib" CACHE PATH "System Lib dir") -set(SYSTEM_LIB64_PATH "/usr/lib64" CACHE PATH "System Lib64 dir") -set(SYSTEM_LOCAL_LIB_PATH "/usr/local/lib" CACHE PATH "System Local Lib dir") -set(SYSTEM_LOCAL_LIB64_PATH "/usr/local/lib64" CACHE PATH "System Local Lib64 dir") -if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - set(SYSTEM_LIB_MULTIARCH_PATH "/usr/lib/x86_64-linux-gnu" CACHE PATH "System Lib MultiArch dir") +set(SYSTEM_LIB_PATH + "/usr/lib" + CACHE PATH "System Lib dir") +set(SYSTEM_LIB64_PATH + "/usr/lib64" + CACHE PATH "System Lib64 dir") +set(SYSTEM_LOCAL_LIB_PATH + "/usr/local/lib" + CACHE PATH "System Local Lib dir") +set(SYSTEM_LOCAL_LIB64_PATH + "/usr/local/lib64" + CACHE PATH "System Local Lib64 dir") +if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") + set(SYSTEM_LIB_MULTIARCH_PATH + "/usr/lib/x86_64-linux-gnu" + CACHE PATH "System Lib MultiArch dir") elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) - set(SYSTEM_LIB_MULTIARCH_PATH "/usr/lib/aarch64-linux-gnu" CACHE PATH "System Lib MultiArch dir") + set(SYSTEM_LIB_MULTIARCH_PATH + "/usr/lib/aarch64-linux-gnu" + CACHE PATH "System Lib MultiArch dir") elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL arm64) - set(SYSTEM_LIB_MULTIARCH_PATH "/usr/lib" CACHE PATH "System Lib MultiArch dir") + set(SYSTEM_LIB_MULTIARCH_PATH + "/usr/lib" + CACHE PATH "System Lib MultiArch dir") else() message(FATAL_ERROR "Unsupported processor type: ${CMAKE_SYSTEM_PROCESSOR}") endif() -if (NOT DEFINED VELOX_HOME) +if(NOT DEFINED VELOX_HOME) set(VELOX_HOME ${GLUTEN_HOME}/ep/build-velox/build/velox_ep) message(STATUS "Set VELOX_HOME to ${VELOX_HOME}") endif() # Keep same compile option with Velox. execute_process( - COMMAND + COMMAND bash -c "( source ${VELOX_HOME}/scripts/setup-helper-functions.sh && echo -n $(get_cxx_flags $ENV{CPU_TARGET}))" - OUTPUT_VARIABLE SCRIPT_CXX_FLAGS - RESULT_VARIABLE COMMAND_STATUS) + OUTPUT_VARIABLE SCRIPT_CXX_FLAGS + RESULT_VARIABLE COMMAND_STATUS) if(COMMAND_STATUS EQUAL "1") - message(FATAL_ERROR "Unable to determine compiler flags!") + message(FATAL_ERROR "Unable to determine compiler flags!") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}") @@ -63,10 +78,14 @@ message("Velox module final CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") # User can specify VELOX_BUILD_PATH, if Velox are built elsewhere. if(NOT DEFINED VELOX_BUILD_PATH) - if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") - set(VELOX_BUILD_PATH "${VELOX_HOME}/_build/debug" CACHE PATH "Velox build directory.") + if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") + set(VELOX_BUILD_PATH + "${VELOX_HOME}/_build/debug" + CACHE PATH "Velox build directory.") else() - set(VELOX_BUILD_PATH "${VELOX_HOME}/_build/release" CACHE PATH "Velox build directory.") + set(VELOX_BUILD_PATH + "${VELOX_HOME}/_build/release" + CACHE PATH "Velox build directory.") endif() endif() @@ -78,25 +97,28 @@ function(ADD_VELOX_DEPENDENCY VELOX_DEP_LIB_NAME VELOX_DEP_LIB_PATH) endif() set(VELOX_DEP_LIB facebook::velox::${VELOX_DEP_LIB_NAME}) add_library(${VELOX_DEP_LIB} STATIC IMPORTED) - set_target_properties(${VELOX_DEP_LIB} PROPERTIES - IMPORTED_LOCATION ${VELOX_DEP_LIB_PATH}) + set_target_properties(${VELOX_DEP_LIB} PROPERTIES IMPORTED_LOCATION + ${VELOX_DEP_LIB_PATH}) target_link_libraries(velox PUBLIC ${VELOX_DEP_LIB}) endfunction() macro(ADD_VELOX_OBJECTS) add_library(velox_objects OBJECT IMPORTED GLOBAL) - set_property(TARGET velox_objects PROPERTY IMPORTED_OBJECTS - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/FileHandle.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConfig.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConnector.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveDataSink.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveDataSource.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HivePartitionUtil.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/PartitionIdGenerator.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/SplitReader.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/TableHandle.cpp.o" - "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConnectorUtil.cpp.o" - ) + set_property( + TARGET velox_objects + PROPERTY + IMPORTED_OBJECTS + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/FileHandle.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConfig.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConnector.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveDataSink.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveDataSource.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HivePartitionUtil.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/PartitionIdGenerator.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/SplitReader.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/TableHandle.cpp.o" + "${VELOX_COMPONENTS_PATH}/connectors/hive/CMakeFiles/velox_hive_connector.dir/HiveConnectorUtil.cpp.o" + ) target_link_libraries(velox PUBLIC velox_objects) endmacro() @@ -112,125 +134,302 @@ endmacro() macro(ADD_VELOX_DEPENDENCIES) add_velox_objects() - add_velox_dependency(expression::sigparser "${VELOX_COMPONENTS_PATH}/expression/signature_parser/libvelox_signature_parser.a") - add_velox_dependency(functions::sparksql::lib "${VELOX_COMPONENTS_PATH}/functions/sparksql/libvelox_functions_spark.a") - add_velox_dependency(functions::sparksql::agg "${VELOX_COMPONENTS_PATH}/functions/sparksql/aggregates/libvelox_functions_spark_aggregates.a") - add_velox_dependency(functions::window::sparksql "${VELOX_COMPONENTS_PATH}/functions/sparksql/window/libvelox_functions_spark_window.a") - add_velox_dependency(functions::prestosql::agg "${VELOX_COMPONENTS_PATH}/functions/prestosql/aggregates/libvelox_aggregates.a") - add_velox_dependency(functions::lib::agg "${VELOX_COMPONENTS_PATH}/functions/lib/aggregates/libvelox_functions_aggregates.a") - add_velox_dependency(functions::prestosql::window "${VELOX_COMPONENTS_PATH}/functions/prestosql/window/libvelox_window.a") - add_velox_dependency(functions::lib::window "${VELOX_COMPONENTS_PATH}/functions/lib/window/libvelox_functions_window.a") - add_velox_dependency(velox::buffer "${VELOX_COMPONENTS_PATH}/buffer/libvelox_buffer.a") - - add_velox_dependency(functions::isnull "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_is_null_functions.a") - add_velox_dependency(functions::prestosql "${VELOX_COMPONENTS_PATH}/functions/prestosql/registration/libvelox_functions_prestosql.a") - add_velox_dependency(functions::prestosql::impl "${VELOX_COMPONENTS_PATH}/functions/prestosql/libvelox_functions_prestosql_impl.a") - add_velox_dependency(functions::json "${VELOX_COMPONENTS_PATH}/functions/prestosql/json/libvelox_functions_json.a") - add_velox_dependency(functions::hyperloglog "${VELOX_COMPONENTS_PATH}/common/hyperloglog/libvelox_common_hyperloglog.a") - add_velox_dependency(functions::lib "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib.a") - add_velox_dependency(functions::lib::date_time_formatter "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib_date_time_formatter.a") + add_velox_dependency( + expression::sigparser + "${VELOX_COMPONENTS_PATH}/expression/signature_parser/libvelox_signature_parser.a" + ) + add_velox_dependency( + functions::sparksql::lib + "${VELOX_COMPONENTS_PATH}/functions/sparksql/libvelox_functions_spark.a") + add_velox_dependency( + functions::sparksql::agg + "${VELOX_COMPONENTS_PATH}/functions/sparksql/aggregates/libvelox_functions_spark_aggregates.a" + ) + add_velox_dependency( + functions::window::sparksql + "${VELOX_COMPONENTS_PATH}/functions/sparksql/window/libvelox_functions_spark_window.a" + ) + add_velox_dependency( + functions::prestosql::agg + "${VELOX_COMPONENTS_PATH}/functions/prestosql/aggregates/libvelox_aggregates.a" + ) + add_velox_dependency( + functions::lib::agg + "${VELOX_COMPONENTS_PATH}/functions/lib/aggregates/libvelox_functions_aggregates.a" + ) + add_velox_dependency( + functions::prestosql::window + "${VELOX_COMPONENTS_PATH}/functions/prestosql/window/libvelox_window.a") + add_velox_dependency( + functions::lib::window + "${VELOX_COMPONENTS_PATH}/functions/lib/window/libvelox_functions_window.a") + add_velox_dependency(velox::buffer + "${VELOX_COMPONENTS_PATH}/buffer/libvelox_buffer.a") + + add_velox_dependency( + functions::isnull + "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_is_null_functions.a") + add_velox_dependency( + functions::prestosql + "${VELOX_COMPONENTS_PATH}/functions/prestosql/registration/libvelox_functions_prestosql.a" + ) + add_velox_dependency( + functions::prestosql::impl + "${VELOX_COMPONENTS_PATH}/functions/prestosql/libvelox_functions_prestosql_impl.a" + ) + add_velox_dependency( + functions::json + "${VELOX_COMPONENTS_PATH}/functions/prestosql/json/libvelox_functions_json.a" + ) + add_velox_dependency( + functions::hyperloglog + "${VELOX_COMPONENTS_PATH}/common/hyperloglog/libvelox_common_hyperloglog.a") + add_velox_dependency( + functions::lib + "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib.a") + add_velox_dependency( + functions::lib::date_time_formatter + "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_lib_date_time_formatter.a" + ) if(BUILD_TESTS) - add_velox_dependency(exec::test "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_exec_test_lib.a") - add_velox_dependency(temp::path "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_temp_path.a") - add_velox_dependency(dwio::common::test::utils "${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a") + add_velox_dependency( + exec::test + "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_exec_test_lib.a") + add_velox_dependency( + temp::path + "${VELOX_COMPONENTS_PATH}/exec/tests/utils/libvelox_temp_path.a") + add_velox_dependency( + dwio::common::test::utils + "${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a" + ) endif() add_velox_dependency(exec "${VELOX_COMPONENTS_PATH}/exec/libvelox_exec.a") if(BUILD_TESTS) - add_velox_dependency(parse::parser "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_parser.a") - add_velox_dependency(duckdb::parser "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_parser.a") - add_velox_dependency(parse::expression "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_expression.a") - add_velox_dependency(parse::utils "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_utils.a") - add_velox_dependency(function::registry "${VELOX_COMPONENTS_PATH}/functions/libvelox_function_registry.a") + add_velox_dependency( + parse::parser "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_parser.a") + add_velox_dependency( + duckdb::parser + "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_parser.a") + add_velox_dependency( + parse::expression + "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_expression.a") + add_velox_dependency( + parse::utils "${VELOX_COMPONENTS_PATH}/parse/libvelox_parse_utils.a") + add_velox_dependency( + function::registry + "${VELOX_COMPONENTS_PATH}/functions/libvelox_function_registry.a") endif() - add_velox_dependency(vector::arrow::bridge "${VELOX_COMPONENTS_PATH}/vector/arrow/libvelox_arrow_bridge.a") + add_velox_dependency( + vector::arrow::bridge + "${VELOX_COMPONENTS_PATH}/vector/arrow/libvelox_arrow_bridge.a") add_velox_dependency(row "${VELOX_COMPONENTS_PATH}/row/libvelox_row_fast.a") - add_velox_dependency(connector "${VELOX_COMPONENTS_PATH}/connectors/libvelox_connector.a") - add_velox_dependency(connector::hive_parition "${VELOX_COMPONENTS_PATH}/connectors/hive/libvelox_hive_partition_function.a") - add_velox_dependency(connector::hive::iceberg::IcebergSplitReader "${VELOX_COMPONENTS_PATH}/connectors/hive/iceberg/libvelox_hive_iceberg_splitreader.a") + add_velox_dependency( + connector "${VELOX_COMPONENTS_PATH}/connectors/libvelox_connector.a") + add_velox_dependency( + connector::hive_parition + "${VELOX_COMPONENTS_PATH}/connectors/hive/libvelox_hive_partition_function.a" + ) + add_velox_dependency( + connector::hive::iceberg::IcebergSplitReader + "${VELOX_COMPONENTS_PATH}/connectors/hive/iceberg/libvelox_hive_iceberg_splitreader.a" + ) if(ENABLE_HDFS) - add_velox_dependency(connector::hive::hdfs "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/hdfs/libvelox_hdfs.a") + add_velox_dependency( + connector::hive::hdfs + "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/hdfs/libvelox_hdfs.a" + ) endif() if(ENABLE_GCS) - add_velox_dependency(connector::hive::gcs "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/gcs/libvelox_gcs.a") + add_velox_dependency( + connector::hive::gcs + "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/gcs/libvelox_gcs.a" + ) endif() if(ENABLE_S3) - add_velox_dependency(connector::hive::s3fs "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/s3fs/libvelox_s3fs.a") + add_velox_dependency( + connector::hive::s3fs + "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/s3fs/libvelox_s3fs.a" + ) endif() if(ENABLE_ABFS) - add_velox_dependency(connector::hive::abfs "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/abfs/libvelox_abfs.a") + add_velox_dependency( + connector::hive::abfs + "${VELOX_COMPONENTS_PATH}/connectors/hive/storage_adapters/abfs/libvelox_abfs.a" + ) endif() - add_velox_dependency(dwio::dwrf::writer "${VELOX_COMPONENTS_PATH}/dwio/dwrf/writer/libvelox_dwio_dwrf_writer.a") - add_velox_dependency(dwio::dwrf::reader "${VELOX_COMPONENTS_PATH}/dwio/dwrf/reader/libvelox_dwio_dwrf_reader.a") - add_velox_dependency(dwio::dwrf::utils "${VELOX_COMPONENTS_PATH}/dwio/dwrf/utils/libvelox_dwio_dwrf_utils.a") - add_velox_dependency(dwio::dwrf::common "${VELOX_COMPONENTS_PATH}/dwio/dwrf/common/libvelox_dwio_dwrf_common.a") - add_velox_dependency(parquet "${VELOX_COMPONENTS_PATH}/dwio/parquet/libvelox_dwio_parquet_reader.a") - add_velox_dependency(parquet::reader::native "${VELOX_COMPONENTS_PATH}/dwio/parquet/reader/libvelox_dwio_native_parquet_reader.a") + add_velox_dependency( + dwio::dwrf::writer + "${VELOX_COMPONENTS_PATH}/dwio/dwrf/writer/libvelox_dwio_dwrf_writer.a") + add_velox_dependency( + dwio::dwrf::reader + "${VELOX_COMPONENTS_PATH}/dwio/dwrf/reader/libvelox_dwio_dwrf_reader.a") + add_velox_dependency( + dwio::dwrf::utils + "${VELOX_COMPONENTS_PATH}/dwio/dwrf/utils/libvelox_dwio_dwrf_utils.a") + add_velox_dependency( + dwio::dwrf::common + "${VELOX_COMPONENTS_PATH}/dwio/dwrf/common/libvelox_dwio_dwrf_common.a") + add_velox_dependency( + parquet + "${VELOX_COMPONENTS_PATH}/dwio/parquet/libvelox_dwio_parquet_reader.a") + add_velox_dependency( + parquet::reader::native + "${VELOX_COMPONENTS_PATH}/dwio/parquet/reader/libvelox_dwio_native_parquet_reader.a" + ) if(BUILD_TESTS) - add_velox_dependency(dwio::common::utils "${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a") - add_velox_dependency(dwio::dwrf::test_utils "${VELOX_COMPONENTS_PATH}/dwio/dwrf/test/utils/libvelox_dwrf_test_utils.a") - add_velox_dependency(parquet::reader::duckdb_conversion "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_conversion.a") + add_velox_dependency( + dwio::common::utils + "${VELOX_COMPONENTS_PATH}/dwio/common/tests/utils/libvelox_dwio_common_test_utils.a" + ) + add_velox_dependency( + dwio::dwrf::test_utils + "${VELOX_COMPONENTS_PATH}/dwio/dwrf/test/utils/libvelox_dwrf_test_utils.a" + ) + add_velox_dependency( + parquet::reader::duckdb_conversion + "${VELOX_COMPONENTS_PATH}/duckdb/conversion/libvelox_duckdb_conversion.a") add_duckdb() - add_velox_dependency(tpch::gen "${VELOX_COMPONENTS_PATH}/tpch/gen/libvelox_tpch_gen.a") - add_velox_dependency(dbgen "${VELOX_COMPONENTS_PATH}/tpch/gen/dbgen/libvelox_dbgen.a") + add_velox_dependency( + tpch::gen "${VELOX_COMPONENTS_PATH}/tpch/gen/libvelox_tpch_gen.a") + add_velox_dependency( + dbgen "${VELOX_COMPONENTS_PATH}/tpch/gen/dbgen/libvelox_dbgen.a") endif() - add_velox_dependency(parquet::reader::thrift "${VELOX_COMPONENTS_PATH}/dwio/parquet/thrift/libvelox_dwio_parquet_thrift.a") - - add_velox_dependency(velox::arrow::parquet::writer "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/libvelox_dwio_arrow_parquet_writer.a") - add_velox_dependency(dwio::arrow::parquet::writer "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/libvelox_dwio_arrow_parquet_writer_lib.a") - add_velox_dependency(dwio::arrow::parquet::writer::util "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/util/libvelox_dwio_arrow_parquet_writer_util_lib.a") - add_velox_dependency(dwio::arrow::parquet::writer::thrift::lib "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/generated/libvelox_dwio_arrow_parquet_writer_thrift_lib.a") - add_velox_dependency(dwio::common::compression "${VELOX_COMPONENTS_PATH}/dwio/common/compression/libvelox_dwio_common_compression.a") - add_velox_dependency(dwio::common "${VELOX_COMPONENTS_PATH}/dwio/common/libvelox_dwio_common.a") - add_velox_dependency(functions::prestosql::types "${VELOX_COMPONENTS_PATH}/functions/prestosql/types/libvelox_presto_types.a") - add_velox_dependency(functions::spark::specialforms "${VELOX_COMPONENTS_PATH}/functions/sparksql/specialforms/libvelox_functions_spark_specialforms.a") - add_velox_dependency(expression "${VELOX_COMPONENTS_PATH}/expression/libvelox_expression.a") + add_velox_dependency( + parquet::reader::thrift + "${VELOX_COMPONENTS_PATH}/dwio/parquet/thrift/libvelox_dwio_parquet_thrift.a" + ) + + add_velox_dependency( + velox::arrow::parquet::writer + "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/libvelox_dwio_arrow_parquet_writer.a" + ) + add_velox_dependency( + dwio::arrow::parquet::writer + "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/libvelox_dwio_arrow_parquet_writer_lib.a" + ) + add_velox_dependency( + dwio::arrow::parquet::writer::util + "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/util/libvelox_dwio_arrow_parquet_writer_util_lib.a" + ) + add_velox_dependency( + dwio::arrow::parquet::writer::thrift::lib + "${VELOX_COMPONENTS_PATH}/dwio/parquet/writer/arrow/generated/libvelox_dwio_arrow_parquet_writer_thrift_lib.a" + ) + add_velox_dependency( + dwio::common::compression + "${VELOX_COMPONENTS_PATH}/dwio/common/compression/libvelox_dwio_common_compression.a" + ) + add_velox_dependency( + dwio::common "${VELOX_COMPONENTS_PATH}/dwio/common/libvelox_dwio_common.a") + add_velox_dependency( + functions::prestosql::types + "${VELOX_COMPONENTS_PATH}/functions/prestosql/types/libvelox_presto_types.a" + ) + add_velox_dependency( + functions::spark::specialforms + "${VELOX_COMPONENTS_PATH}/functions/sparksql/specialforms/libvelox_functions_spark_specialforms.a" + ) + add_velox_dependency( + expression "${VELOX_COMPONENTS_PATH}/expression/libvelox_expression.a") add_velox_dependency(core "${VELOX_COMPONENTS_PATH}/core/libvelox_core.a") - add_velox_dependency(type::fbhive "${VELOX_COMPONENTS_PATH}/type/fbhive/libvelox_type_fbhive.a") + add_velox_dependency( + type::fbhive "${VELOX_COMPONENTS_PATH}/type/fbhive/libvelox_type_fbhive.a") add_velox_dependency(type "${VELOX_COMPONENTS_PATH}/type/libvelox_type.a") - add_velox_dependency(vector::serializes "${VELOX_COMPONENTS_PATH}/serializers/libvelox_presto_serializer.a") - add_velox_dependency(functions::lib::util "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_util.a") - add_velox_dependency(vector "${VELOX_COMPONENTS_PATH}/vector/libvelox_vector.a") - add_velox_dependency(expression::function "${VELOX_COMPONENTS_PATH}/expression/libvelox_expression_functions.a") - add_velox_dependency(expression::type_calculation "${VELOX_COMPONENTS_PATH}/expression/type_calculation/libvelox_type_calculation.a") - - add_velox_dependency(common::caching "${VELOX_COMPONENTS_PATH}/common/caching/libvelox_caching.a") - add_velox_dependency(common::base "${VELOX_COMPONENTS_PATH}/common/base/libvelox_common_base.a") - add_velox_dependency(common::memory "${VELOX_COMPONENTS_PATH}/common/memory/libvelox_memory.a") - add_velox_dependency(common::serialization "${VELOX_COMPONENTS_PATH}/common/serialization/libvelox_serialization.a") - add_velox_dependency(common::base::exception "${VELOX_COMPONENTS_PATH}/common/base/libvelox_exception.a") - - add_velox_dependency(type::tz "${VELOX_COMPONENTS_PATH}/type/tz/libvelox_type_tz.a") - add_velox_dependency(dwio::dwrf::proto "${VELOX_COMPONENTS_PATH}/dwio/dwrf/proto/libvelox_dwio_dwrf_proto.a") - add_velox_dependency(dwio::catalog::fbhive "${VELOX_COMPONENTS_PATH}/dwio/catalog/fbhive/libvelox_dwio_catalog_fbhive.a") - add_velox_dependency(dwio::common::exception "${VELOX_COMPONENTS_PATH}/dwio/common/exception/libvelox_dwio_common_exception.a") - add_velox_dependency(dwio::common::encryption "${VELOX_COMPONENTS_PATH}/dwio/common/encryption/libvelox_dwio_common_encryption.a") - - add_velox_dependency(core::config "${VELOX_COMPONENTS_PATH}/core/libvelox_config.a") - add_velox_dependency(common::encode "${VELOX_COMPONENTS_PATH}/common/encode/libvelox_encode.a") - add_velox_dependency(common::time "${VELOX_COMPONENTS_PATH}/common/time/libvelox_time.a") + add_velox_dependency( + vector::serializes + "${VELOX_COMPONENTS_PATH}/serializers/libvelox_presto_serializer.a") + add_velox_dependency( + functions::lib::util + "${VELOX_COMPONENTS_PATH}/functions/lib/libvelox_functions_util.a") + add_velox_dependency(vector + "${VELOX_COMPONENTS_PATH}/vector/libvelox_vector.a") + add_velox_dependency( + expression::function + "${VELOX_COMPONENTS_PATH}/expression/libvelox_expression_functions.a") + add_velox_dependency( + expression::type_calculation + "${VELOX_COMPONENTS_PATH}/expression/type_calculation/libvelox_type_calculation.a" + ) + + add_velox_dependency( + common::caching + "${VELOX_COMPONENTS_PATH}/common/caching/libvelox_caching.a") + add_velox_dependency( + common::base "${VELOX_COMPONENTS_PATH}/common/base/libvelox_common_base.a") + add_velox_dependency( + common::memory "${VELOX_COMPONENTS_PATH}/common/memory/libvelox_memory.a") + add_velox_dependency( + common::serialization + "${VELOX_COMPONENTS_PATH}/common/serialization/libvelox_serialization.a") + add_velox_dependency( + common::base::exception + "${VELOX_COMPONENTS_PATH}/common/base/libvelox_exception.a") + + add_velox_dependency(type::tz + "${VELOX_COMPONENTS_PATH}/type/tz/libvelox_type_tz.a") + add_velox_dependency( + dwio::dwrf::proto + "${VELOX_COMPONENTS_PATH}/dwio/dwrf/proto/libvelox_dwio_dwrf_proto.a") + add_velox_dependency( + dwio::catalog::fbhive + "${VELOX_COMPONENTS_PATH}/dwio/catalog/fbhive/libvelox_dwio_catalog_fbhive.a" + ) + add_velox_dependency( + dwio::common::exception + "${VELOX_COMPONENTS_PATH}/dwio/common/exception/libvelox_dwio_common_exception.a" + ) + add_velox_dependency( + dwio::common::encryption + "${VELOX_COMPONENTS_PATH}/dwio/common/encryption/libvelox_dwio_common_encryption.a" + ) + + add_velox_dependency(core::config + "${VELOX_COMPONENTS_PATH}/core/libvelox_config.a") + add_velox_dependency( + common::encode "${VELOX_COMPONENTS_PATH}/common/encode/libvelox_encode.a") + add_velox_dependency(common::time + "${VELOX_COMPONENTS_PATH}/common/time/libvelox_time.a") if(BUILD_TESTS) - add_velox_dependency(common::file::test "${VELOX_COMPONENTS_PATH}/common/file/tests/libvelox_file_test_utils.a") + add_velox_dependency( + common::file::test + "${VELOX_COMPONENTS_PATH}/common/file/tests/libvelox_file_test_utils.a") endif() - add_velox_dependency(common::file "${VELOX_COMPONENTS_PATH}/common/file/libvelox_file.a") - add_velox_dependency(common::process "${VELOX_COMPONENTS_PATH}/common/process/libvelox_process.a") - - add_velox_dependency(common::test_util "${VELOX_COMPONENTS_PATH}/common/testutil/libvelox_test_util.a") - - add_velox_dependency(external::md5 "${VELOX_COMPONENTS_PATH}/external/md5/libmd5.a") - add_velox_dependency(external::date "${VELOX_COMPONENTS_PATH}/external/date/libvelox_external_date.a") - add_velox_dependency(velox::parquet::writer "${VELOX_COMPONENTS_PATH}/dwio/parquet/libvelox_dwio_parquet_writer.a") + add_velox_dependency(common::file + "${VELOX_COMPONENTS_PATH}/common/file/libvelox_file.a") + add_velox_dependency( + common::process + "${VELOX_COMPONENTS_PATH}/common/process/libvelox_process.a") + + add_velox_dependency( + common::test_util + "${VELOX_COMPONENTS_PATH}/common/testutil/libvelox_test_util.a") + + add_velox_dependency(external::md5 + "${VELOX_COMPONENTS_PATH}/external/md5/libmd5.a") + add_velox_dependency( + external::date + "${VELOX_COMPONENTS_PATH}/external/date/libvelox_external_date.a") + add_velox_dependency( + velox::parquet::writer + "${VELOX_COMPONENTS_PATH}/dwio/parquet/libvelox_dwio_parquet_writer.a") if(BUILD_TESTS) - add_velox_dependency(vector::test::util "${VELOX_COMPONENTS_PATH}/vector/tests/utils/libvelox_vector_test_lib.a") + add_velox_dependency( + vector::test::util + "${VELOX_COMPONENTS_PATH}/vector/tests/utils/libvelox_vector_test_lib.a") endif() - add_velox_dependency(common::compression "${VELOX_COMPONENTS_PATH}/common/compression/libvelox_common_compression.a") - add_velox_dependency(common::io "${VELOX_COMPONENTS_PATH}/common/io/libvelox_common_io.a") - add_velox_dependency(velox::status "${VELOX_COMPONENTS_PATH}/common/base/libvelox_status.a") + add_velox_dependency( + common::compression + "${VELOX_COMPONENTS_PATH}/common/compression/libvelox_common_compression.a") + add_velox_dependency( + common::io "${VELOX_COMPONENTS_PATH}/common/io/libvelox_common_io.a") + add_velox_dependency(velox::status + "${VELOX_COMPONENTS_PATH}/common/base/libvelox_status.a") endmacro() macro(find_libhdfs3) @@ -241,18 +440,16 @@ macro(find_libhdfs3) find_path(libhdfs3_INCLUDE_DIR hdfs/hdfs.h) set(CMAKE_FIND_LIBRARY_SUFFIXES ".so") find_library(libhdfs3_LIBRARY NAMES hdfs3) - find_package_handle_standard_args(libhdfs3 DEFAULT_MSG - libhdfs3_INCLUDE_DIR - libhdfs3_LIBRARY - ) + find_package_handle_standard_args(libhdfs3 DEFAULT_MSG libhdfs3_INCLUDE_DIR + libhdfs3_LIBRARY) add_library(HDFS::hdfs3 SHARED IMPORTED) - set_target_properties(HDFS::hdfs3 PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${libhdfs3_INCLUDE_DIR}" - IMPORTED_LOCATION "${libhdfs3_LIBRARY}" - ) + set_target_properties( + HDFS::hdfs3 + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${libhdfs3_INCLUDE_DIR}" + IMPORTED_LOCATION "${libhdfs3_LIBRARY}") endif() - if (NOT libhdfs3_FOUND) + if(NOT libhdfs3_FOUND) message(FATAL_ERROR "LIBHDFS3 Library Not Found") endif() endmacro() @@ -262,30 +459,35 @@ macro(find_re2) if(re2_FOUND AND TARGET re2::re2) set(RE2_LIBRARY re2::re2) else() - find_library(RE2_LIBRARY NAMES re2 PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} NO_DEFAULT_PATH) + find_library( + RE2_LIBRARY + NAMES re2 + PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} + ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} + NO_DEFAULT_PATH) endif() - if (NOT RE2_LIBRARY) - message(FATAL_ERROR "RE2 Library Not Found") + if(NOT RE2_LIBRARY) + message(FATAL_ERROR "RE2 Library Not Found") else() message(STATUS "RE2 Library Can Be Found in ${RE2_LIBRARY}") endif() endmacro() macro(find_awssdk) - set (CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_package(AWSSDK REQUIRED COMPONENTS s3;identity-management) endmacro() macro(find_gcssdk) - set (CMAKE_FIND_LIBRARY_SUFFIXES ".so") + set(CMAKE_FIND_LIBRARY_SUFFIXES ".so") find_package(google_cloud_cpp_storage REQUIRED) endmacro() macro(find_azure) find_package(CURL REQUIRED) find_package(LibXml2 REQUIRED) - set (CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") find_package(azure-storage-blobs-cpp CONFIG REQUIRED) find_package(azure-storage-files-datalake-cpp CONFIG REQUIRED) endmacro() @@ -327,12 +529,11 @@ set(VELOX_SRCS utils/VeloxArrowUtils.cc utils/ConfigExtractor.cc utils/Common.cc - utils/VeloxBatchAppender.cc - ) + utils/VeloxBatchAppender.cc) -if (ENABLE_HDFS) +if(ENABLE_HDFS) list(APPEND VELOX_SRCS utils/HdfsUtils.cc) -endif () +endif() if(ENABLE_S3) find_package(ZLIB) @@ -346,30 +547,30 @@ add_library(velox SHARED ${VELOX_SRCS}) if(ENABLE_GLUTEN_VCPKG) # Hide symbols of static dependencies - target_link_options(velox PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) + target_link_options( + velox PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) endif() -target_include_directories(velox PUBLIC - ${CMAKE_SYSTEM_INCLUDE_PATH} - ${JNI_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} - ${VELOX_HOME}/ - ${VELOX_BUILD_PATH}/ - ${VELOX_BUILD_PATH}/_deps/xsimd-src/include/ - ${VELOX_HOME}/third_party/xsimd/include/) - -set_target_properties(velox PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${root_directory}/releases - ) - -## If folly is not installed in system lib paths, please add -## `-DCMAKE_PREFIX_PATH="${folly lib path}" to cmake arguments. -## It is also applicable to other dependencies. +target_include_directories( + velox + PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} + ${JNI_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR} + ${VELOX_HOME}/ + ${VELOX_BUILD_PATH}/ + ${VELOX_BUILD_PATH}/_deps/xsimd-src/include/ + ${VELOX_HOME}/third_party/xsimd/include/) + +set_target_properties(velox PROPERTIES LIBRARY_OUTPUT_DIRECTORY + ${root_directory}/releases) + +# If folly is not installed in system lib paths, please add +# `-DCMAKE_PREFIX_PATH="${folly lib path}" to cmake arguments. It is also +# applicable to other dependencies. find_package(Folly REQUIRED CONFIG) -target_include_directories(velox PUBLIC - ${GTEST_INCLUDE_DIRS} - ${PROTOBUF_INCLUDE}) +target_include_directories(velox PUBLIC ${GTEST_INCLUDE_DIRS} + ${PROTOBUF_INCLUDE}) target_link_libraries(velox PUBLIC gluten) add_velox_dependencies() @@ -383,11 +584,13 @@ target_link_libraries(velox PUBLIC Folly::folly) find_re2() target_link_libraries(velox PUBLIC ${RE2_LIBRARY}) -# since https://github.com/facebookincubator/velox/commit/47970417ac92135e862c0fde350d4d60fa2f1423 +# since +# https://github.com/facebookincubator/velox/commit/47970417ac92135e862c0fde350d4d60fa2f1423 if(Stemmer_FOUND) target_link_libraries(velox PUBLIC stemmer::stemmer) else() - add_velox_dependency(velox "${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a") + add_velox_dependency( + velox "${VELOX_BUILD_PATH}/_deps/libstemmer/src/libstemmer/libstemmer.a") endif() set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) @@ -396,7 +599,8 @@ find_package(simdjson CONFIG) if(simdjson_FOUND AND TARGET simdjson::simdjson) target_link_libraries(velox PUBLIC simdjson::simdjson) else() - add_velox_dependency(external::simdjson "${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a") + add_velox_dependency(external::simdjson + "${VELOX_BUILD_PATH}/_deps/simdjson-build/libsimdjson.a") endif() set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) @@ -411,7 +615,8 @@ endif() if(Thrift_FOUND) target_link_libraries(velox PUBLIC thrift::thrift) else() - add_velox_dependency(thrift "${ARROW_HOME}/src/arrow_ep-build/thrift_ep-install/lib/libthrift.a") + add_velox_dependency( + thrift "${ARROW_HOME}/src/arrow_ep-build/thrift_ep-install/lib/libthrift.a") endif() if(BUILD_TESTS) @@ -451,7 +656,13 @@ if(ENABLE_ABFS) target_link_libraries(velox PUBLIC Azure::azure-storage-files-datalake) endif() -add_custom_command(TARGET velox POST_BUILD COMMAND ld $ || true - COMMENT "Checking ld result of libvelox.so") -add_custom_command(TARGET velox POST_BUILD COMMAND ldd $ || true - COMMENT "Checking ldd result of libvelox.so") +add_custom_command( + TARGET velox + POST_BUILD + COMMAND ld $ || true + COMMENT "Checking ld result of libvelox.so") +add_custom_command( + TARGET velox + POST_BUILD + COMMAND ldd $ || true + COMMENT "Checking ldd result of libvelox.so") diff --git a/cpp/velox/benchmarks/CMakeLists.txt b/cpp/velox/benchmarks/CMakeLists.txt index 74f21c29bc1d..903ec0d65825 100644 --- a/cpp/velox/benchmarks/CMakeLists.txt +++ b/cpp/velox/benchmarks/CMakeLists.txt @@ -15,10 +15,15 @@ find_arrow_lib(${PARQUET_LIB_NAME}) -set(VELOX_BENCHMARK_COMMON_SRCS common/FileReaderIterator.cc common/BenchmarkUtils.cc) +set(VELOX_BENCHMARK_COMMON_SRCS common/FileReaderIterator.cc + common/BenchmarkUtils.cc) add_library(velox_benchmark_common STATIC ${VELOX_BENCHMARK_COMMON_SRCS}) -target_include_directories(velox_benchmark_common PUBLIC ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/core) -target_link_libraries(velox_benchmark_common PUBLIC Arrow::parquet velox benchmark::benchmark google::glog) +target_include_directories( + velox_benchmark_common PUBLIC ${CMAKE_SOURCE_DIR}/velox + ${CMAKE_SOURCE_DIR}/core) +target_link_libraries( + velox_benchmark_common PUBLIC Arrow::parquet velox benchmark::benchmark + google::glog) function(add_velox_benchmark BM_EXEC BM_FILE) add_executable(${BM_EXEC} ${BM_FILE}) diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt index 29beb69da220..f3d65f127f67 100644 --- a/cpp/velox/tests/CMakeLists.txt +++ b/cpp/velox/tests/CMakeLists.txt @@ -16,14 +16,9 @@ function(add_velox_test TEST_EXEC) set(options) set(one_value_args) - set(multi_value_args - SOURCES - ) - cmake_parse_arguments(ARG - "${options}" - "${one_value_args}" - "${multi_value_args}" - ${ARGN}) + set(multi_value_args SOURCES) + cmake_parse_arguments(ARG "${options}" "${one_value_args}" + "${multi_value_args}" ${ARGN}) if(ARG_SOURCES) set(SOURCES ${ARG_SOURCES}) @@ -31,34 +26,34 @@ function(add_velox_test TEST_EXEC) message(FATAL_ERROR "No sources specified for test ${TEST_NAME}") endif() add_executable(${TEST_EXEC} ${SOURCES} ${VELOX_TEST_COMMON_SRCS}) - target_include_directories(${TEST_EXEC} PRIVATE ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/src ${VELOX_BUILD_PATH}/_deps/duckdb-src/src/include) - target_link_libraries(${TEST_EXEC} velox_benchmark_common GTest::gtest GTest::gtest_main) + target_include_directories( + ${TEST_EXEC} PRIVATE ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/src + ${VELOX_BUILD_PATH}/_deps/duckdb-src/src/include) + target_link_libraries(${TEST_EXEC} velox_benchmark_common GTest::gtest + GTest::gtest_main) gtest_discover_tests(${TEST_EXEC} DISCOVERY_MODE PRE_TEST) endfunction() set(VELOX_TEST_COMMON_SRCS JsonToProtoConverter.cc FilePathGenerator.cc) add_velox_test(velox_shuffle_writer_test SOURCES VeloxShuffleWriterTest.cc) -# TODO: ORC is not well supported. -# add_velox_test(orc_test SOURCES OrcTest.cc) +# TODO: ORC is not well supported. add_velox_test(orc_test SOURCES OrcTest.cc) add_velox_test( - velox_operators_test - SOURCES - VeloxColumnarToRowTest.cc - VeloxRowToColumnarTest.cc - VeloxColumnarBatchSerializerTest.cc - VeloxColumnarBatchTest.cc) + velox_operators_test SOURCES VeloxColumnarToRowTest.cc + VeloxRowToColumnarTest.cc VeloxColumnarBatchSerializerTest.cc + VeloxColumnarBatchTest.cc) add_velox_test( - velox_plan_conversion_test - SOURCES - Substrait2VeloxPlanConversionTest.cc - Substrait2VeloxPlanValidatorTest.cc - Substrait2VeloxValuesNodeConversionTest.cc - SubstraitExtensionCollectorTest.cc - VeloxSubstraitRoundTripTest.cc - VeloxSubstraitSignatureTest.cc - VeloxToSubstraitTypeTest.cc) -add_velox_test(spark_functions_test SOURCES SparkFunctionTest.cc FunctionTest.cc) + velox_plan_conversion_test + SOURCES + Substrait2VeloxPlanConversionTest.cc + Substrait2VeloxPlanValidatorTest.cc + Substrait2VeloxValuesNodeConversionTest.cc + SubstraitExtensionCollectorTest.cc + VeloxSubstraitRoundTripTest.cc + VeloxSubstraitSignatureTest.cc + VeloxToSubstraitTypeTest.cc) +add_velox_test(spark_functions_test SOURCES SparkFunctionTest.cc + FunctionTest.cc) add_velox_test(execution_ctx_test SOURCES RuntimeTest.cc) add_velox_test(velox_memory_test SOURCES MemoryManagerTest.cc) add_velox_test(buffer_outputstream_test SOURCES BufferOutputStreamTest.cc) diff --git a/docs/developers/CppCodingStyle.md b/docs/developers/CppCodingStyle.md index 9dca4cf69fbc..42101882a9e5 100644 --- a/docs/developers/CppCodingStyle.md +++ b/docs/developers/CppCodingStyle.md @@ -28,11 +28,20 @@ Gluten CPP coding, there are a few Philosophical rules as the following. ## Code Formatting Many aspects of C++ coding style will be covered by clang-format, such as spacing, -line width, indentation and ordering (for includes, using directives and etc).  +line width, indentation and ordering (for includes, using directives and etc). * Always ensure your code is compatible with clang-format-15 for Velox backend. * `dev/formatcppcode.sh` is provided for formatting Velox CPP code. +To format cmake files like CMakeLists.txt & *.cmake, `cmake-format` is required to +be installed. Here is an example. + +``` +apt install python3-pip -y +pip3 install --user cmake-format +cmake-format --first-comment-is-literal True --in-place cpp/velox/CMakeLists.txt +``` + ## Naming Conventions * Use **PascalCase** for types (class, struct, enum, type alias, type From c4566ebc1bb4264f14858b313d8565f4268ff3ff Mon Sep 17 00:00:00 2001 From: WangGuangxin Date: Fri, 14 Jun 2024 08:55:38 +0800 Subject: [PATCH 264/402] [GLUTEN-5965][VL] Support the pushdown "NOT IN" filter (#5966) --- .../gluten/execution/TestOperator.scala | 65 +++++++++++ cpp/velox/substrait/SubstraitToVeloxPlan.cc | 110 +++++++++++++----- cpp/velox/substrait/SubstraitToVeloxPlan.h | 23 +++- 3 files changed, 169 insertions(+), 29 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 3cf485aac06b..a892b6f313a4 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -157,6 +157,71 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla checkLengthAndPlan(df, 60141) } + test("not in") { + // integral type + val df = runQueryAndCompare( + "select l_orderkey from lineitem " + + "where l_partkey not in (1552, 674, 1062)") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + checkLengthAndPlan(df, 60053) + + val df2 = runQueryAndCompare( + "select l_orderkey from lineitem " + + "where l_partkey not in (1552, 674) and l_partkey not in (1062)") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + checkLengthAndPlan(df2, 60053) + + val df3 = runQueryAndCompare( + "select l_orderkey from lineitem " + + "where l_partkey not in (1552, 674) and l_partkey != 1062") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + checkLengthAndPlan(df3, 60053) + + // string type + val df4 = + runQueryAndCompare("select o_orderstatus from orders where o_orderstatus not in ('O', 'F')") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + checkLengthAndPlan(df4, 363) + + // bool type + withTable("t") { + sql("create table t (id int, b boolean) using parquet") + sql("insert into t values (1, true), (2, false), (3, null)") + runQueryAndCompare("select * from t where b not in (true)") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + + runQueryAndCompare("select * from t where b not in (true, false)") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + } + + // mix not-in with range + runQueryAndCompare( + "select l_orderkey from lineitem " + + "where l_partkey not in (1552, 674) and l_partkey >= 1552") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + + // mix not-in with in + runQueryAndCompare( + "select l_orderkey from lineitem " + + "where l_partkey not in (1552, 674) and l_partkey in (1552)") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + + // not-in with or relation + runQueryAndCompare( + "select l_orderkey from lineitem " + + "where l_partkey not in (1552, 674) or l_partkey in (1552)") { + checkGlutenOperatorMatch[FileSourceScanExecTransformer] + } + } + test("coalesce") { var df = runQueryAndCompare( "select l_orderkey, coalesce(l_comment, 'default_val') " + diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 4e875d4790e5..8b8a9262403c 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -20,6 +20,7 @@ #include "VariantToVectorConverter.h" #include "velox/connectors/hive/HiveDataSink.h" #include "velox/exec/TableWriter.h" +#include "velox/type/Filter.h" #include "velox/type/Type.h" #include "utils/ConfigExtractor.h" @@ -1465,10 +1466,12 @@ connector::hive::SubfieldFilters SubstraitToVeloxPlanConverter::createSubfieldFi auto expr = scalarFunction.arguments()[0].value(); if (expr.has_scalar_function()) { // Set its child to filter info with reverse enabled. - setFilterInfo(scalarFunction.arguments()[0].value().scalar_function(), inputTypeList, columnToFilterInfo, true); + setFilterInfo(expr.scalar_function(), inputTypeList, columnToFilterInfo, true); + } else if (expr.has_singular_or_list()) { + auto singularOrList = expr.singular_or_list(); + setFilterInfo(singularOrList, columnToFilterInfo, true); } else { - // TODO: support push down of Not In. - VELOX_NYI("Scalar function expected."); + VELOX_NYI("Only support push down Not with scalar function or In."); } } else if (filterName == sOr) { VELOX_CHECK(scalarFunction.arguments().size() == 2); @@ -1593,24 +1596,26 @@ bool SubstraitToVeloxPlanConverter::canPushdownNot( std::vector& rangeRecorders) { VELOX_CHECK(scalarFunction.arguments().size() == 1, "Only one arg is expected for Not."); const auto& notArg = scalarFunction.arguments()[0]; - if (!notArg.value().has_scalar_function()) { - // Not for a Boolean Literal or Or List is not supported curretly. - // It can be pushed down with an AlwaysTrue or AlwaysFalse Range. - return false; - } - - auto argFunction = - SubstraitParser::findFunctionSpec(functionMap_, notArg.value().scalar_function().function_reference()); - auto functionName = SubstraitParser::getNameBeforeDelimiter(argFunction); + if (notArg.value().has_singular_or_list()) { + auto singularOrList = notArg.value().singular_or_list(); + if (!canPushdownSingularOrList(singularOrList)) { + return false; + } + uint32_t colIdx = getColumnIndexFromSingularOrList(singularOrList); + return rangeRecorders.at(colIdx).setInRange(); + } else if (notArg.value().has_scalar_function()) { + auto argFunction = + SubstraitParser::findFunctionSpec(functionMap_, notArg.value().scalar_function().function_reference()); + auto functionName = SubstraitParser::getNameBeforeDelimiter(argFunction); - static const std::unordered_set supportedNotFunctions = {sGte, sGt, sLte, sLt, sEqual}; + static const std::unordered_set supportedNotFunctions = {sGte, sGt, sLte, sLt, sEqual}; - uint32_t fieldIdx; - bool isFieldOrWithLiteral = fieldOrWithLiteral(notArg.value().scalar_function().arguments(), fieldIdx); + uint32_t fieldIdx; + bool isFieldOrWithLiteral = fieldOrWithLiteral(notArg.value().scalar_function().arguments(), fieldIdx); - if (supportedNotFunctions.find(functionName) != supportedNotFunctions.end() && isFieldOrWithLiteral && - rangeRecorders.at(fieldIdx).setCertainRangeForFunction(functionName, true /*reverse*/)) { - return true; + return ( + supportedNotFunctions.find(functionName) != supportedNotFunctions.end() && isFieldOrWithLiteral && + rangeRecorders.at(fieldIdx).setCertainRangeForFunction(functionName, true /*reverse*/)); } return false; } @@ -1966,6 +1971,7 @@ template void SubstraitToVeloxPlanConverter::setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters) {} @@ -1973,6 +1979,7 @@ template <> void SubstraitToVeloxPlanConverter::setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters) { std::vector values; @@ -1981,13 +1988,18 @@ void SubstraitToVeloxPlanConverter::setInFilter( int64_t value = variant.value(); values.emplace_back(value); } - filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + if (negated) { + filters[common::Subfield(inputName)] = common::createNegatedBigintValues(values, nullAllowed); + } else { + filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + } } template <> void SubstraitToVeloxPlanConverter::setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters) { // Use bigint values for int type. @@ -1998,13 +2010,18 @@ void SubstraitToVeloxPlanConverter::setInFilter( int64_t value = variant.value(); values.emplace_back(value); } - filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + if (negated) { + filters[common::Subfield(inputName)] = common::createNegatedBigintValues(values, nullAllowed); + } else { + filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + } } template <> void SubstraitToVeloxPlanConverter::setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters) { // Use bigint values for small int type. @@ -2015,13 +2032,18 @@ void SubstraitToVeloxPlanConverter::setInFilter( int64_t value = variant.value(); values.emplace_back(value); } - filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + if (negated) { + filters[common::Subfield(inputName)] = common::createNegatedBigintValues(values, nullAllowed); + } else { + filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + } } template <> void SubstraitToVeloxPlanConverter::setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters) { // Use bigint values for tiny int type. @@ -2032,13 +2054,18 @@ void SubstraitToVeloxPlanConverter::setInFilter( int64_t value = variant.value(); values.emplace_back(value); } - filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + if (negated) { + filters[common::Subfield(inputName)] = common::createNegatedBigintValues(values, nullAllowed); + } else { + filters[common::Subfield(inputName)] = common::createBigintValues(values, nullAllowed); + } } template <> void SubstraitToVeloxPlanConverter::setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters) { std::vector values; @@ -2047,7 +2074,11 @@ void SubstraitToVeloxPlanConverter::setInFilter( std::string value = variant.value(); values.emplace_back(value); } - filters[common::Subfield(inputName)] = std::make_unique(values, nullAllowed); + if (negated) { + filters[common::Subfield(inputName)] = std::make_unique(values, nullAllowed); + } else { + filters[common::Subfield(inputName)] = std::make_unique(values, nullAllowed); + } } template @@ -2102,6 +2133,17 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( if (filterInfo.notValue_) { filters[common::Subfield(inputName)] = std::make_unique(!filterInfo.notValue_.value().value(), nullAllowed); + } else if (filterInfo.notValues_.size() > 0) { + std::set notValues; + for (auto v : filterInfo.notValues_) { + notValues.emplace(v.value()); + } + if (notValues.size() == 1) { + filters[common::Subfield(inputName)] = std::make_unique(!(*notValues.begin()), nullAllowed); + } else { + // if there are more than one distinct value in NOT IN list, the filter should be AlwaysFalse + filters[common::Subfield(inputName)] = std::make_unique(); + } } else if (rangeSize == 0) { // IsNull/IsNotNull. if (!nullAllowed) { @@ -2140,11 +2182,22 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters( if (filterInfo.values_.size() > 0) { // To filter out null is a default behaviour of Spark IN expression. nullAllowed = false; - setInFilter(filterInfo.values_, nullAllowed, inputName, filters); + setInFilter(filterInfo.values_, nullAllowed, false, inputName, filters); // Currently, In cannot coexist with other filter conditions // due to multirange is in 'OR' relation but 'AND' is needed. VELOX_CHECK(rangeSize == 0, "LowerBounds or upperBounds conditons cannot be supported after IN filter."); VELOX_CHECK(!filterInfo.notValue_.has_value(), "Not equal cannot be supported after IN filter."); + VELOX_CHECK(filterInfo.notValues_.size() == 0, "Not in cannot be supported after IN filter."); + return; + } + + // Handle not in filter. + if (filterInfo.notValues_.size() > 0) { + setInFilter(filterInfo.notValues_, filterInfo.nullAllowed_, true, inputName, filters); + // Currently, NOT In cannot coexist with other filter conditions + // due to multirange is in 'OR' relation but 'AND' is needed. + VELOX_CHECK(rangeSize == 0, "LowerBounds or upperBounds conditons cannot be supported after NOT IN filter."); + VELOX_CHECK(!filterInfo.notValue_.has_value(), "Not equal cannot be supported after NOT IN filter."); return; } @@ -2429,7 +2482,8 @@ uint32_t SubstraitToVeloxPlanConverter::getColumnIndexFromSingularOrList( void SubstraitToVeloxPlanConverter::setFilterInfo( const ::substrait::Expression_SingularOrList& singularOrList, - std::vector& columnToFilterInfo) { + std::vector& columnToFilterInfo, + bool reverse) { VELOX_CHECK(singularOrList.options_size() > 0, "At least one option is expected."); // Get the column index. uint32_t colIdx = getColumnIndexFromSingularOrList(singularOrList); @@ -2443,7 +2497,11 @@ void SubstraitToVeloxPlanConverter::setFilterInfo( variants.emplace_back(exprConverter_->toVeloxExpr(option.literal())->value()); } // Set the value list to filter info. - columnToFilterInfo[colIdx].setValues(variants); + if (!reverse) { + columnToFilterInfo[colIdx].setValues(variants); + } else { + columnToFilterInfo[colIdx].setNotValues(variants); + } } } // namespace gluten diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.h b/cpp/velox/substrait/SubstraitToVeloxPlan.h index 3a0e677afeaa..1535b1f85f51 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.h +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.h @@ -377,6 +377,16 @@ class SubstraitToVeloxPlanConverter { } } + // Set a list of values to be used in the push down of 'not in' expression. + void setNotValues(const std::vector& notValues) { + for (const auto& value : notValues) { + notValues_.emplace_back(value); + } + if (!initialized_) { + initialized_ = true; + } + } + // Whether this filter map is initialized. bool initialized_ = false; @@ -402,6 +412,9 @@ class SubstraitToVeloxPlanConverter { // The list of values used in 'in' expression. std::vector values_; + + // The list of values should not be equal to. + std::vector notValues_; }; /// Returns unique ID to use for plan node. Produces sequential numbers @@ -464,9 +477,11 @@ class SubstraitToVeloxPlanConverter { bool reverse = false); /// Extract SingularOrList and set it to the filter info map. + /// If reverse is true, the opposite filter info will be set. void setFilterInfo( const ::substrait::Expression_SingularOrList& singularOrList, - std::vector& columnToFilterInfo); + std::vector& columnToFilterInfo, + bool reverse = false); /// Extract SingularOrList and returns the field index. static uint32_t getColumnIndexFromSingularOrList(const ::substrait::Expression_SingularOrList&); @@ -484,13 +499,15 @@ class SubstraitToVeloxPlanConverter { template void createNotEqualFilter(variant notVariant, bool nullAllowed, std::vector>& colFilters); - /// Create a values range to handle in filter. - /// variants: the list of values extracted from the in expression. + /// Create a values range to handle (not) in filter. + /// variants: the list of values extracted from the (not) in expression. + // negated: false for IN filter, true for NOT IN filter. /// inputName: the column input name. template void setInFilter( const std::vector& variants, bool nullAllowed, + bool negated, const std::string& inputName, connector::hive::SubfieldFilters& filters); From 68ffdff91d884608ea646cb0d2dcc415484f71ef Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Fri, 14 Jun 2024 11:23:44 +0800 Subject: [PATCH 265/402] [VL] Fix undefined symbol with qat (#6081) --- cpp/CMake/BuildQATzip.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/CMake/BuildQATzip.cmake b/cpp/CMake/BuildQATzip.cmake index c68ef25ada2e..fd75757d7286 100644 --- a/cpp/CMake/BuildQATzip.cmake +++ b/cpp/CMake/BuildQATzip.cmake @@ -76,12 +76,16 @@ macro(build_qatzip) "${OSAL_LIBRARY}" Threads::Threads) + # Fix libudev.so not get linked. + set(QATZIP_LINK_OPTIONS "-Wl,--no-as-needed") + add_library(qatzip::qatzip STATIC IMPORTED) set_target_properties( qatzip::qatzip PROPERTIES IMPORTED_LOCATION "${QATZIP_STATIC_LIB_TARGETS}" INTERFACE_INCLUDE_DIRECTORIES "${QATZIP_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${QATZIP_LINK_LIBRARIES}") + INTERFACE_LINK_LIBRARIES "${QATZIP_LINK_LIBRARIES}" + INTERFACE_LINK_OPTIONS "${QATZIP_LINK_OPTIONS}") add_dependencies(qatzip::qatzip qatzip_ep) endmacro() From 2316dae3c915d7b4f1b71b117564882287e5c5d7 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:28:32 +0800 Subject: [PATCH 266/402] [VL] Daily Update Velox Version (2024_06_14) (#6084) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 5840f21c251b..49654d8a8cd5 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_13 +VELOX_BRANCH=2024_06_14 VELOX_HOME="" #Set on run gluten on HDFS From 1065dd3d2cd8733358fd8cad531be16190b0b12f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 13:34:27 +0800 Subject: [PATCH 267/402] [VL] Fix inaccurate calculation of task slot number used by s.g.s.c.b.v.IOThreads (#6071) --- .../gluten/utils/VeloxBloomFilterTest.java | 87 +++++++++++-------- cpp/velox/compute/VeloxBackend.cc | 3 + .../org/apache/gluten/GlutenPlugin.scala | 5 +- .../org/apache/gluten/GlutenConfig.scala | 29 +++++-- .../integration/action/Parameterized.scala | 2 +- 5 files changed, 78 insertions(+), 48 deletions(-) diff --git a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java index ba349a4f04b4..fda4003ddd20 100644 --- a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java @@ -16,11 +16,13 @@ */ package org.apache.gluten.utils; +import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.ListenerApi; import org.apache.gluten.backendsapi.velox.VeloxListenerApi; import com.codahale.metrics.MetricRegistry; import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; import org.apache.spark.api.plugin.PluginContext; import org.apache.spark.resource.ResourceInformation; import org.apache.spark.util.TaskResources$; @@ -33,50 +35,13 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.Map; public class VeloxBloomFilterTest { - @BeforeClass public static void setup() { final ListenerApi api = new VeloxListenerApi(); - PluginContext pluginContext = - new PluginContext() { - @Override - public MetricRegistry metricRegistry() { - return null; - } - - @Override - public SparkConf conf() { - return new SparkConf(); - } - - @Override - public String executorID() { - return ""; - } - - @Override - public String hostname() { - return ""; - } - - @Override - public Map resources() { - return Collections.emptyMap(); - } - - @Override - public void send(Object message) throws IOException {} - - @Override - public Object ask(Object message) throws Exception { - return null; - } - }; - api.onDriverStart(null, pluginContext); + api.onDriverStart(mockSparkContext(), mockPluginContext()); } @Test @@ -226,4 +191,50 @@ private static void checkFalsePositives(BloomFilter filter, int start) { Assert.assertTrue(negativeFalsePositives > 0); Assert.assertTrue(negativeFalsePositives < attemptCount); } + + private static SparkContext mockSparkContext() { + // Not yet implemented. + return null; + } + + private static PluginContext mockPluginContext() { + return new PluginContext() { + @Override + public MetricRegistry metricRegistry() { + throw new UnsupportedOperationException(); + } + + @Override + public SparkConf conf() { + final SparkConf conf = new SparkConf(); + conf.set(GlutenConfig.GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY(), "0"); + return conf; + } + + @Override + public String executorID() { + throw new UnsupportedOperationException(); + } + + @Override + public String hostname() { + throw new UnsupportedOperationException(); + } + + @Override + public Map resources() { + throw new UnsupportedOperationException(); + } + + @Override + public void send(Object message) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Object ask(Object message) throws Exception { + throw new UnsupportedOperationException(); + } + }; + } } diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 10d1c7529710..1ec5879966d6 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -226,6 +226,9 @@ void VeloxBackend::initConnector() { FLAGS_cache_prefetch_min_pct = backendConf_->get(kCachePrefetchMinPct, 0); auto ioThreads = backendConf_->get(kVeloxIOThreads, kVeloxIOThreadsDefault); + GLUTEN_CHECK( + ioThreads >= 0, + kVeloxIOThreads + " was set to negative number " + std::to_string(ioThreads) + ", this should not happen."); if (ioThreads > 0) { ioExecutor_ = std::make_unique(ioThreads); } diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 7c601e48d4f6..cafed66eb8f0 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -162,8 +162,9 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { // task slots val taskSlots = SparkResourceUtil.getTaskSlots(conf) + conf.set(GlutenConfig.GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY, taskSlots.toString) - var onHeapSize: Long = + val onHeapSize: Long = if (conf.contains(GlutenConfig.GLUTEN_ONHEAP_SIZE_KEY)) { conf.getSizeAsBytes(GlutenConfig.GLUTEN_ONHEAP_SIZE_KEY) } else { @@ -175,7 +176,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { // size. Otherwise, the off-heap size is set to the value specified by the user (if any). // Note that this means that we will IGNORE the off-heap size specified by the user if the // dynamic off-heap feature is enabled. - var offHeapSize: Long = + val offHeapSize: Long = if (conf.getBoolean(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED, false)) { // Since when dynamic off-heap sizing is enabled, we commingle on-heap // and off-heap memory, we set the off-heap size to the usable on-heap size. We will diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 2376a1f39c1e..13ad8e47113b 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -241,6 +241,12 @@ class GlutenConfig(conf: SQLConf) extends Logging { def memoryIsolation: Boolean = conf.getConf(COLUMNAR_MEMORY_ISOLATION) + def numTaskSlotsPerExecutor: Int = { + val numSlots = conf.getConf(NUM_TASK_SLOTS_PER_EXECUTOR) + assert(numSlots > 0, s"Number of task slot not found. This should not happen.") + numSlots + } + def offHeapMemorySize: Long = conf.getConf(COLUMNAR_OFFHEAP_SIZE_IN_BYTES) def taskOffHeapMemorySize: Long = conf.getConf(COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES) @@ -271,7 +277,9 @@ class GlutenConfig(conf: SQLConf) extends Logging { def veloxSsdODirectEnabled: Boolean = conf.getConf(COLUMNAR_VELOX_SSD_ODIRECT_ENABLED) - def veloxConnectorIOThreads: Integer = conf.getConf(COLUMNAR_VELOX_CONNECTOR_IO_THREADS) + def veloxConnectorIOThreads: Int = { + conf.getConf(COLUMNAR_VELOX_CONNECTOR_IO_THREADS).getOrElse(numTaskSlotsPerExecutor) + } def veloxSplitPreloadPerDriver: Integer = conf.getConf(COLUMNAR_VELOX_SPLIT_PRELOAD_PER_DRIVER) @@ -533,6 +541,7 @@ object GlutenConfig { val GLUTEN_DEBUG_KEEP_JNI_WORKSPACE = "spark.gluten.sql.debug.keepJniWorkspace" // Added back to Spark Conf during executor initialization + val GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY = "spark.gluten.numTaskSlotsPerExecutor" val GLUTEN_OFFHEAP_SIZE_IN_BYTES_KEY = "spark.gluten.memory.offHeap.size.in.bytes" val GLUTEN_TASK_OFFHEAP_SIZE_IN_BYTES_KEY = "spark.gluten.memory.task.offHeap.size.in.bytes" val GLUTEN_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES_KEY = @@ -678,7 +687,7 @@ object GlutenConfig { (SPARK_S3_IAM_SESSION_NAME, ""), ( COLUMNAR_VELOX_CONNECTOR_IO_THREADS.key, - COLUMNAR_VELOX_CONNECTOR_IO_THREADS.defaultValueString), + conf.getOrElse(GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY, "-1")), (COLUMNAR_SHUFFLE_CODEC.key, ""), (COLUMNAR_SHUFFLE_CODEC_BACKEND.key, ""), ("spark.hadoop.input.connect.timeout", "180000"), @@ -1165,6 +1174,16 @@ object GlutenConfig { .stringConf .createOptional + val NUM_TASK_SLOTS_PER_EXECUTOR = + buildConf(GlutenConfig.GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY) + .internal() + .doc( + "Must provide default value since non-execution operations " + + "(e.g. org.apache.spark.sql.Dataset#summary) doesn't propagate configurations using " + + "org.apache.spark.sql.execution.SQLExecution#withSQLConfPropagated") + .intConf + .createWithDefaultString("-1") + val COLUMNAR_OFFHEAP_SIZE_IN_BYTES = buildConf(GlutenConfig.GLUTEN_OFFHEAP_SIZE_IN_BYTES_KEY) .internal() @@ -1303,11 +1322,7 @@ object GlutenConfig { .doc("The Size of the IO thread pool in the Connector. This thread pool is used for split" + " preloading and DirectBufferedInput.") .intConf - .createWithDefaultFunction( - () => - SQLConf.get.getConfString("spark.executor.cores", "1").toInt / SQLConf.get - .getConfString("spark.task.cpus", "1") - .toInt) + .createOptional val COLUMNAR_VELOX_ASYNC_TIMEOUT = buildStaticConf("spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping") diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala index e2fc526ce566..74f22a05f5fe 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala @@ -342,7 +342,7 @@ object Parameterized { desc: String): Unit = { println(s"Warming up: Running query: $id...") try { - val testDesc = "Gluten Spark %s %s warm up".format(desc, id) + val testDesc = "Gluten Spark %s [%s] Warm Up".format(desc, id) sessionSwitcher.useSession("test", testDesc) runner.createTables(creator, sessionSwitcher.spark()) val result = runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain = false) From d18ab8f3560b3d0bb02e4d23014f52269db279e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Fri, 14 Jun 2024 13:37:10 +0800 Subject: [PATCH 268/402] [CORE] Remove getPartitionFilters from scanTransformer (#6076) --- .../gluten/execution/FileSourceScanExecTransformer.scala | 2 -- .../apache/spark/sql/hive/HiveTableScanExecTransformer.scala | 4 ---- 2 files changed, 6 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala index c3c296c13c3c..4f120488c2fb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/FileSourceScanExecTransformer.scala @@ -102,8 +102,6 @@ abstract class FileSourceScanExecTransformerBase( override def getMetadataColumns(): Seq[AttributeReference] = metadataColumns - def getPartitionFilters(): Seq[Expression] = partitionFilters - override def outputAttributes(): Seq[Attribute] = output override def getPartitions: Seq[InputPartition] = { diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala index 5dfa85b269a8..95793e5dc935 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveTableScanExecTransformer.scala @@ -188,10 +188,6 @@ object HiveTableScanExecTransformer { plan.isInstanceOf[HiveTableScanExec] } - def getPartitionFilters(plan: SparkPlan): Seq[Expression] = { - plan.asInstanceOf[HiveTableScanExec].partitionPruningPred - } - def copyWith(plan: SparkPlan, newPartitionFilters: Seq[Expression]): SparkPlan = { val hiveTableScanExec = plan.asInstanceOf[HiveTableScanExec] hiveTableScanExec.copy(partitionPruningPred = newPartitionFilters)(sparkSession = From e52bda689ea212ce957fbb7d4ff9be6cad2523ea Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Fri, 14 Jun 2024 13:37:55 +0800 Subject: [PATCH 269/402] [GLUTEN-6067][CH][Part 1] Support CH backend with Spark3.5 (#6068) * [GLUTEN-6067][CH] Support CH backend with Spark3.5 (Task 1 and Taks 2) Support CH backend with Spark3.5: 1. Upgrade Spark version to 3.5 and compile passed; Done 2. Upgrade Delta version to 3.2 and compile passed; Done 3. CH backend UT passed: (now only the MergeTree + Delta UT passed); 4. Parquet native write passed; 5. Gluten UT passed; 6. Support to run Gluten CH CI with Spark 3.5 --- backends-clickhouse/pom.xml | 33 + .../ClickhouseOptimisticTransaction.scala | 16 +- .../spark/sql/delta/DeltaAdapter.scala} | 8 +- .../sql/delta/catalog/ClickHouseTableV2.scala | 164 +-- .../OptimizeTableCommandOverwrites.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../ClickhouseOptimisticTransaction.scala | 183 +++ .../apache/spark/sql/delta/DeltaAdapter.scala | 21 + .../sql/delta/catalog/ClickHouseTableV2.scala | 186 +++ .../OptimizeTableCommandOverwrites.scala | 333 +++++ .../v2/clickhouse/ClickHouseDataSource.scala | 145 +++ .../clickhouse/ClickHouseSparkCatalog.scala | 662 ++++++++++ .../io/delta/tables/ClickhouseTable.scala | 134 ++ .../ClickhouseOptimisticTransaction.scala | 193 +++ .../apache/spark/sql/delta/DeltaAdapter.scala | 21 + .../org/apache/spark/sql/delta/DeltaLog.scala | 1106 +++++++++++++++++ .../org/apache/spark/sql/delta/Snapshot.scala | 663 ++++++++++ .../sql/delta/catalog/ClickHouseTableV2.scala | 220 ++++ .../sql/delta/commands/DeleteCommand.scala | 557 +++++++++ .../delta/commands/OptimizeTableCommand.scala | 608 +++++++++ .../OptimizeTableCommandOverwrites.scala | 345 +++++ .../sql/delta/commands/UpdateCommand.scala | 556 +++++++++ .../sql/delta/commands/VacuumCommand.scala | 735 +++++++++++ .../commands/merge/ClassicMergeExecutor.scala | 571 +++++++++ .../delta/files/MergeTreeCommitProtocol.scala | 255 ++++ .../v2/clickhouse/ClickHouseDataSource.scala | 144 +++ .../clickhouse/ClickHouseSparkCatalog.scala | 734 +++++++++++ .../source/DeltaMergeTreeFileFormat.scala | 133 ++ .../clickhouse/CHIteratorApi.scala | 2 +- .../clickhouse/CHSparkPlanExecApi.scala | 2 +- .../gluten/utils/CHInputPartitionsUtil.scala | 18 +- .../spark/sql/delta/DeltaAdapterTrait.scala} | 7 +- .../delta/catalog/ClickHouseTableV2Base.scala | 185 +++ .../v1/CHMergeTreeWriterInjects.scala | 4 +- .../MergeTreeFileFormatWriter.scala | 7 +- .../sql/execution/utils/CHExecUtil.scala | 9 +- .../sql/execution/utils/PushDownUtil.scala | 30 +- ...utenClickHouseDeltaParquetWriteSuite.scala | 54 +- ...utenClickHouseMergeTreeOptimizeSuite.scala | 8 +- ...ickHouseMergeTreePathBasedWriteSuite.scala | 17 +- .../GlutenClickHouseMergeTreeWriteSuite.scala | 11 +- ...utenClickHouseTPCHParquetBucketSuite.scala | 2 +- .../expressions/aggregate/CustomSum.scala | 10 +- .../apache/gluten/sql/shims/SparkShims.scala | 20 +- .../sql/shims/spark32/Spark32Shims.scala | 21 + .../sql/shims/spark33/Spark33Shims.scala | 21 + .../sql/shims/spark34/Spark34Shims.scala | 20 + .../sql/shims/spark35/Spark35Shims.scala | 22 +- 49 files changed, 8915 insertions(+), 281 deletions(-) rename backends-clickhouse/src/main/{scala => delta-20}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (92%) rename backends-clickhouse/src/main/{delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala => delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala} (78%) rename backends-clickhouse/src/main/{scala => delta-20}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (55%) rename backends-clickhouse/src/main/{scala => delta-20}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/src/main/{scala => delta-20}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/src/main/{scala => delta-20}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala create mode 100644 backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala create mode 100644 backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala create mode 100644 backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename backends-clickhouse/src/main/{delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala => scala/org/apache/spark/sql/delta/DeltaAdapterTrait.scala} (79%) create mode 100644 backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2Base.scala diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index a37734b150bb..27ebd75f2d59 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -215,6 +215,38 @@ 8.5.9 test + + org.apache.arrow + arrow-memory-core + ${arrow.version} + provided + + + io.netty + netty-common + + + io.netty + netty-buffer + + + + + org.apache.arrow + arrow-vector + ${arrow.version} + provided + + + io.netty + netty-common + + + io.netty + netty-buffer + + + @@ -272,6 +304,7 @@ src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/*.scala + src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/merge/*.scala src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/files/*.scala src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/DeltaLog.scala src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 92% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index d59467e11293..0794b45158e6 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -32,7 +32,6 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker} import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat import org.apache.spark.util.{Clock, SerializableConfiguration} import org.apache.commons.lang3.exception.ExceptionUtils @@ -139,20 +138,7 @@ class ClickhouseOptimisticTransaction( MergeTreeFileFormatWriter.write( sparkSession = spark, plan = newQueryPlan, - fileFormat = new DeltaMergeTreeFileFormat( - metadata, - tableV2.dataBaseName, - tableV2.tableName, - ClickhouseSnapshot.genSnapshotId(tableV2.snapshot), - tableV2.orderByKeyOption, - tableV2.lowCardKeyOption, - tableV2.minmaxIndexKeyOption, - tableV2.bfIndexKeyOption, - tableV2.setIndexKeyOption, - tableV2.primaryKeyOption, - tableV2.clickhouseTableConfigs, - tableV2.partitionColumns - ), + fileFormat = tableV2.getFileFormat(metadata), // formats. committer = committer, outputSpec = outputSpec, diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 78% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala index dd6ad383e0d7..b6d4c04844c4 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala @@ -14,10 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.delta -import org.apache.spark.sql.delta.{DeltaLog, Snapshot} - -object DeltaLogAdapter { - def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.unsafeVolatileSnapshot +object DeltaAdapter extends DeltaAdapterTrait { + override def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.snapshot } diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 55% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index d5cd4f984ca6..90370f0b1d99 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -17,11 +17,11 @@ package org.apache.spark.sql.delta.catalog import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} +import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} -import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTimeTravelSpec} +import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTimeTravelSpec, Snapshot} import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table import org.apache.spark.sql.delta.sources.DeltaDataSource @@ -54,8 +54,8 @@ class ClickHouseTableV2( tableIdentifier, timeTravelOpt, options, - cdcOptions) { - protected def getMetadata: Metadata = if (snapshot == null) Metadata() else snapshot.metadata + cdcOptions) + with ClickHouseTableV2Base { lazy val (rootPath, partitionFilters, timeTravelByPath) = { if (catalogTable.isDefined) { @@ -93,126 +93,6 @@ class ClickHouseTableV2( new WriteIntoDeltaBuilder(deltaLog, info.options) } - lazy val dataBaseName = catalogTable - .map(_.identifier.database.getOrElse("default")) - .getOrElse("clickhouse") - - lazy val tableName = catalogTable - .map(_.identifier.table) - .getOrElse(path.toUri.getPath) - - lazy val bucketOption: Option[BucketSpec] = { - val tableProperties = properties() - if (tableProperties.containsKey("numBuckets")) { - val numBuckets = tableProperties.get("numBuckets").trim.toInt - val bucketColumnNames: Seq[String] = - tableProperties.get("bucketColumnNames").split(",").map(_.trim).toSeq - val sortColumnNames: Seq[String] = if (tableProperties.containsKey("orderByKey")) { - tableProperties.get("orderByKey").split(",").map(_.trim).toSeq - } else Seq.empty[String] - Some(BucketSpec(numBuckets, bucketColumnNames, sortColumnNames)) - } else { - None - } - } - - lazy val lowCardKeyOption: Option[Seq[String]] = { - getCommaSeparatedColumns("lowCardKey") - } - - lazy val minmaxIndexKeyOption: Option[Seq[String]] = { - getCommaSeparatedColumns("minmaxIndexKey") - } - - lazy val bfIndexKeyOption: Option[Seq[String]] = { - getCommaSeparatedColumns("bloomfilterIndexKey") - } - - lazy val setIndexKeyOption: Option[Seq[String]] = { - getCommaSeparatedColumns("setIndexKey") - } - - private def getCommaSeparatedColumns(keyName: String) = { - val tableProperties = properties() - if (tableProperties.containsKey(keyName)) { - if (tableProperties.get(keyName).nonEmpty) { - val keys = tableProperties.get(keyName).split(",").map(_.trim).toSeq - keys.foreach( - s => { - if (s.contains(".")) { - throw new IllegalStateException( - s"$keyName $s can not contain '.' (not support nested column yet)") - } - }) - Some(keys.map(s => s.toLowerCase())) - } else { - None - } - } else { - None - } - } - - lazy val orderByKeyOption: Option[Seq[String]] = { - if (bucketOption.isDefined && bucketOption.get.sortColumnNames.nonEmpty) { - val orderByKes = bucketOption.get.sortColumnNames - val invalidKeys = orderByKes.intersect(partitionColumns) - if (invalidKeys.nonEmpty) { - throw new IllegalStateException( - s"partition cols $invalidKeys can not be in the order by keys.") - } - Some(orderByKes) - } else { - val tableProperties = properties() - if (tableProperties.containsKey("orderByKey")) { - if (tableProperties.get("orderByKey").nonEmpty) { - val orderByKes = tableProperties.get("orderByKey").split(",").map(_.trim).toSeq - val invalidKeys = orderByKes.intersect(partitionColumns) - if (invalidKeys.nonEmpty) { - throw new IllegalStateException( - s"partition cols $invalidKeys can not be in the order by keys.") - } - Some(orderByKes) - } else { - None - } - } else { - None - } - } - } - - lazy val primaryKeyOption: Option[Seq[String]] = { - if (orderByKeyOption.isDefined) { - val tableProperties = properties() - if (tableProperties.containsKey("primaryKey")) { - if (tableProperties.get("primaryKey").nonEmpty) { - val primaryKeys = tableProperties.get("primaryKey").split(",").map(_.trim).toSeq - if (!orderByKeyOption.get.mkString(",").startsWith(primaryKeys.mkString(","))) { - throw new IllegalStateException( - s"Primary key $primaryKeys must be a prefix of the sorting key") - } - Some(primaryKeys) - } else { - None - } - } else { - None - } - } else { - None - } - } - - lazy val partitionColumns = snapshot.metadata.partitionColumns - - lazy val clickhouseTableConfigs: Map[String, String] = { - val tableProperties = properties() - val configs = scala.collection.mutable.Map[String, String]() - configs += ("storage_policy" -> tableProperties.getOrDefault("storage_policy", "default")) - configs.toMap - } - def getFileFormat(meta: Metadata): DeltaMergeTreeFileFormat = { new DeltaMergeTreeFileFormat( meta, @@ -230,41 +110,19 @@ class ClickHouseTableV2( ) } - def cacheThis(): Unit = { - deltaLog2Table.put(deltaLog, this) - } + override def deltaProperties(): ju.Map[String, String] = properties() - cacheThis() + override def deltaCatalog(): Option[CatalogTable] = catalogTable - def primaryKey(): String = primaryKeyOption match { - case Some(keys) => keys.mkString(",") - case None => "" - } - - def orderByKey(): String = orderByKeyOption match { - case Some(keys) => keys.mkString(",") - case None => "tuple()" - } - - def lowCardKey(): String = lowCardKeyOption match { - case Some(keys) => keys.mkString(",") - case None => "" - } + override def deltaPath(): Path = path - def minmaxIndexKey(): String = minmaxIndexKeyOption match { - case Some(keys) => keys.mkString(",") - case None => "" - } + override def deltaSnapshot(): Snapshot = snapshot - def bfIndexKey(): String = bfIndexKeyOption match { - case Some(keys) => keys.mkString(",") - case None => "" + def cacheThis(): Unit = { + deltaLog2Table.put(deltaLog, this) } - def setIndexKey(): String = setIndexKeyOption match { - case Some(keys) => keys.mkString(",") - case None => "" - } + cacheThis() } @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala new file mode 100644 index 000000000000..0794b45158e6 --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta + +import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.execution.ColumnarToRowExecBase + +import org.apache.spark.SparkException +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 +import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} +import org.apache.spark.sql.delta.files.MergeTreeCommitProtocol +import org.apache.spark.sql.delta.schema.InvariantViolationException +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec +import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker} +import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.util.{Clock, SerializableConfiguration} + +import org.apache.commons.lang3.exception.ExceptionUtils + +import scala.collection.mutable.ListBuffer + +object ClickhouseOptimisticTransaction {} +class ClickhouseOptimisticTransaction( + override val deltaLog: DeltaLog, + override val snapshot: Snapshot)(implicit override val clock: Clock) + extends OptimisticTransaction(deltaLog, snapshot) { + + def this(deltaLog: DeltaLog, snapshotOpt: Option[Snapshot] = None)(implicit clock: Clock) { + this( + deltaLog, + snapshotOpt.getOrElse(deltaLog.update()) + ) + } + + def insertFakeRowAdaptor(queryPlan: SparkPlan): SparkPlan = queryPlan match { + // if the child is columnar, we can just wrap&transfer the columnar data + case c2r: ColumnarToRowExecBase => + FakeRowAdaptor(c2r.child) + // If the child is aqe, we make aqe "support columnar", + // then aqe itself will guarantee to generate columnar outputs. + // So FakeRowAdaptor will always consumes columnar data, + // thus avoiding the case of c2r->aqe->r2c->writer + case aqe: AdaptiveSparkPlanExec => + FakeRowAdaptor( + AdaptiveSparkPlanExec( + aqe.inputPlan, + aqe.context, + aqe.preprocessingRules, + aqe.isSubquery, + supportsColumnar = true + )) + case other => FakeRowAdaptor(other) + } + + override def writeFiles( + inputData: Dataset[_], + writeOptions: Option[DeltaOptions], + additionalConstraints: Seq[Constraint]): Seq[FileAction] = { + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + hasWritten = true + + val spark = inputData.sparkSession + val (data, partitionSchema) = performCDCPartition(inputData) + val outputPath = deltaLog.dataPath + + val (queryExecution, output, generatedColumnConstraints, _) = + normalizeData(deltaLog, data) + val partitioningColumns = getPartitioningColumns(partitionSchema, output) + + val committer = new MergeTreeCommitProtocol("delta-mergetree", outputPath.toString, None) + + // val (optionalStatsTracker, _) = + // getOptionalStatsTrackerAndStatsCollection(output, outputPath, partitionSchema, data) + val (optionalStatsTracker, _) = (None, None) + + val constraints = + Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints + + SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { + val outputSpec = FileFormatWriter.OutputSpec(outputPath.toString, Map.empty, output) + + val queryPlan = queryExecution.executedPlan + val newQueryPlan = insertFakeRowAdaptor(queryPlan) + + val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() + + if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { + val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( + new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), + BasicWriteJobStatsTracker.metrics) + // registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) + statsTrackers.append(basicWriteJobStatsTracker) + } + + // Retain only a minimal selection of Spark writer options to avoid any potential + // compatibility issues + var options = writeOptions match { + case None => Map.empty[String, String] + case Some(writeOptions) => + writeOptions.options.filterKeys { + key => + key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || + key.equalsIgnoreCase(DeltaOptions.COMPRESSION) + }.toMap + } + + spark.conf.getAll.foreach( + entry => { + if ( + entry._1.startsWith(s"${CHBackendSettings.getBackendConfigPrefix}.runtime_settings") + || entry._1.equalsIgnoreCase(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key) + ) { + options += (entry._1 -> entry._2) + } + }) + + try { + val tableV2 = ClickHouseTableV2.getTable(deltaLog) + MergeTreeFileFormatWriter.write( + sparkSession = spark, + plan = newQueryPlan, + fileFormat = tableV2.getFileFormat(metadata), + // formats. + committer = committer, + outputSpec = outputSpec, + // scalastyle:off deltahadoopconfiguration + hadoopConf = spark.sessionState + .newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), + // scalastyle:on deltahadoopconfiguration + orderByKeyOption = tableV2.orderByKeyOption, + lowCardKeyOption = tableV2.lowCardKeyOption, + minmaxIndexKeyOption = tableV2.minmaxIndexKeyOption, + bfIndexKeyOption = tableV2.bfIndexKeyOption, + setIndexKeyOption = tableV2.setIndexKeyOption, + primaryKeyOption = tableV2.primaryKeyOption, + partitionColumns = partitioningColumns, + bucketSpec = tableV2.bucketOption, + statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers, + options = options, + constraints = constraints + ) + } catch { + case s: SparkException => + // Pull an InvariantViolationException up to the top level if it was the root cause. + val violationException = ExceptionUtils.getRootCause(s) + if (violationException.isInstanceOf[InvariantViolationException]) { + throw violationException + } else { + throw s + } + } + } + committer.addedStatuses.toSeq ++ committer.changeFiles + } else { + // TODO: support native delta parquet write + // 1. insert FakeRowAdaptor + // 2. DeltaInvariantCheckerExec transform + // 3. DeltaTaskStatisticsTracker collect null count / min values / max values + // 4. set the parameters 'staticPartitionWriteOnly', 'isNativeAppliable', + // 'nativeFormat' in the LocalProperty of the sparkcontext + super.writeFiles(inputData, writeOptions, additionalConstraints) + } + } +} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala new file mode 100644 index 000000000000..8a9c5585e888 --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta + +object DeltaAdapter extends DeltaAdapterTrait { + override def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.unsafeVolatileSnapshot +} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala new file mode 100644 index 000000000000..90370f0b1d99 --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta.catalog +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.connector.read.InputPartition +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} +import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTimeTravelSpec, Snapshot} +import org.apache.spark.sql.delta.actions.Metadata +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table +import org.apache.spark.sql.delta.sources.DeltaDataSource +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} +import org.apache.spark.sql.execution.datasources.utils.MergeTreePartsPartitionsUtil +import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.util.collection.BitSet + +import org.apache.hadoop.fs.Path + +import java.{util => ju} + +import scala.collection.JavaConverters._ + +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) +class ClickHouseTableV2( + override val spark: SparkSession, + override val path: Path, + override val catalogTable: Option[CatalogTable] = None, + override val tableIdentifier: Option[String] = None, + override val timeTravelOpt: Option[DeltaTimeTravelSpec] = None, + override val options: Map[String, String] = Map.empty, + override val cdcOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(), + val clickhouseExtensionOptions: Map[String, String] = Map.empty) + extends DeltaTableV2( + spark, + path, + catalogTable, + tableIdentifier, + timeTravelOpt, + options, + cdcOptions) + with ClickHouseTableV2Base { + + lazy val (rootPath, partitionFilters, timeTravelByPath) = { + if (catalogTable.isDefined) { + // Fast path for reducing path munging overhead + (new Path(catalogTable.get.location), Nil, None) + } else { + DeltaDataSource.parsePathIdentifier(spark, path.toString, options) + } + } + + private lazy val timeTravelSpec: Option[DeltaTimeTravelSpec] = { + if (timeTravelOpt.isDefined && timeTravelByPath.isDefined) { + throw DeltaErrors.multipleTimeTravelSyntaxUsed + } + timeTravelOpt.orElse(timeTravelByPath) + } + + override def name(): String = + catalogTable + .map(_.identifier.unquotedString) + .orElse(tableIdentifier) + .getOrElse(s"clickhouse.`${deltaLog.dataPath}`") + + override def properties(): ju.Map[String, String] = { + val ret = super.properties() + + // for file path based write + if (snapshot.version < 0 && clickhouseExtensionOptions.nonEmpty) { + ret.putAll(clickhouseExtensionOptions.asJava) + } + ret + } + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + new WriteIntoDeltaBuilder(deltaLog, info.options) + } + + def getFileFormat(meta: Metadata): DeltaMergeTreeFileFormat = { + new DeltaMergeTreeFileFormat( + meta, + dataBaseName, + tableName, + ClickhouseSnapshot.genSnapshotId(snapshot), + orderByKeyOption, + lowCardKeyOption, + minmaxIndexKeyOption, + bfIndexKeyOption, + setIndexKeyOption, + primaryKeyOption, + clickhouseTableConfigs, + partitionColumns + ) + } + + override def deltaProperties(): ju.Map[String, String] = properties() + + override def deltaCatalog(): Option[CatalogTable] = catalogTable + + override def deltaPath(): Path = path + + override def deltaSnapshot(): Snapshot = snapshot + + def cacheThis(): Unit = { + deltaLog2Table.put(deltaLog, this) + } + + cacheThis() +} + +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) +class TempClickHouseTableV2( + override val spark: SparkSession, + override val catalogTable: Option[CatalogTable] = None) + extends ClickHouseTableV2(spark, null, catalogTable) { + import collection.JavaConverters._ + override def properties(): ju.Map[String, String] = catalogTable.get.properties.asJava + override lazy val partitionColumns: Seq[String] = catalogTable.get.partitionColumnNames + override def cacheThis(): Unit = {} +} + +object ClickHouseTableV2 extends Logging { + private val deltaLog2Table = + new scala.collection.concurrent.TrieMap[DeltaLog, ClickHouseTableV2]() + // for CTAS use + val temporalThreadLocalCHTable = new ThreadLocal[ClickHouseTableV2]() + + def getTable(deltaLog: DeltaLog): ClickHouseTableV2 = { + if (deltaLog2Table.contains(deltaLog)) { + deltaLog2Table(deltaLog) + } else if (temporalThreadLocalCHTable.get() != null) { + temporalThreadLocalCHTable.get() + } else { + throw new IllegalStateException( + s"Can not find ClickHouseTableV2 for deltalog ${deltaLog.dataPath}") + } + } + + def clearCache(): Unit = { + deltaLog2Table.clear() + temporalThreadLocalCHTable.remove() + } + + def partsPartitions( + deltaLog: DeltaLog, + relation: HadoopFsRelation, + selectedPartitions: Array[PartitionDirectory], + output: Seq[Attribute], + bucketedScan: Boolean, + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + disableBucketedScan: Boolean, + filterExprs: Seq[Expression]): Seq[InputPartition] = { + val tableV2 = ClickHouseTableV2.getTable(deltaLog) + + MergeTreePartsPartitionsUtil.getMergeTreePartsPartitions( + relation, + selectedPartitions, + output, + bucketedScan, + tableV2.spark, + tableV2, + optionalBucketSet, + optionalNumCoalescedBuckets, + disableBucketedScan, + filterExprs) + + } +} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala new file mode 100644 index 000000000000..f7a180b6a239 --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta.commands + +import org.apache.gluten.expression.ConverterUtils + +import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException} +import org.apache.spark.internal.Logging +import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage +import org.apache.spark.internal.io.SparkHadoopWriterUtils +import org.apache.spark.shuffle.FetchFailedException +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddFile, FileAction} +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 +import org.apache.spark.sql.errors.QueryExecutionErrors +import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper +import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.v1.clickhouse._ +import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StructType +import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} + +import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} +import org.apache.hadoop.mapreduce.{TaskAttemptContext, TaskAttemptID, TaskID, TaskType} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl + +import java.util.{Date, UUID} + +import scala.collection.mutable.ArrayBuffer + +object OptimizeTableCommandOverwrites extends Logging { + + case class TaskDescription( + path: String, + database: String, + tableName: String, + snapshotId: String, + orderByKeyOption: Option[Seq[String]], + lowCardKeyOption: Option[Seq[String]], + minmaxIndexKeyOption: Option[Seq[String]], + bfIndexKeyOption: Option[Seq[String]], + setIndexKeyOption: Option[Seq[String]], + primaryKeyOption: Option[Seq[String]], + partitionColumns: Seq[String], + partList: Seq[String], + tableSchema: StructType, + clickhouseTableConfigs: Map[String, String], + serializableHadoopConf: SerializableConfiguration, + jobIdInstant: Long, + partitionDir: Option[String], + bucketDir: Option[String] + ) + + private def executeTask( + description: TaskDescription, + sparkStageId: Int, + sparkPartitionId: Int, + sparkAttemptNumber: Int + ): MergeTreeWriteTaskResult = { + + val jobId = SparkHadoopWriterUtils.createJobID(new Date(description.jobIdInstant), sparkStageId) + val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId) + val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber) + + // Set up the attempt context required to use in the output committer. + val taskAttemptContext: TaskAttemptContext = { + // Set up the configuration object + val hadoopConf = description.serializableHadoopConf.value + hadoopConf.set("mapreduce.job.id", jobId.toString) + hadoopConf.set("mapreduce.task.id", taskAttemptId.getTaskID.toString) + hadoopConf.set("mapreduce.task.attempt.id", taskAttemptId.toString) + hadoopConf.setBoolean("mapreduce.task.ismap", true) + hadoopConf.setInt("mapreduce.task.partition", 0) + + new TaskAttemptContextImpl(hadoopConf, taskAttemptId) + } + + try { + Utils.tryWithSafeFinallyAndFailureCallbacks(block = { + + val uuid = UUID.randomUUID.toString + + val planWithSplitInfo = CHMergeTreeWriterInjects.genMergeTreeWriteRel( + description.path, + description.database, + description.tableName, + description.snapshotId, + description.orderByKeyOption, + description.lowCardKeyOption, + description.minmaxIndexKeyOption, + description.bfIndexKeyOption, + description.setIndexKeyOption, + description.primaryKeyOption, + description.partitionColumns, + description.partList, + ConverterUtils.convertNamedStructJson(description.tableSchema), + description.clickhouseTableConfigs, + description.tableSchema.toAttributes + ) + + val datasourceJniWrapper = new CHDatasourceJniWrapper() + val returnedMetrics = + datasourceJniWrapper.nativeMergeMTParts( + planWithSplitInfo.plan, + planWithSplitInfo.splitInfo, + uuid, + taskId.getId.toString, + description.partitionDir.getOrElse(""), + description.bucketDir.getOrElse("") + ) + if (returnedMetrics != null && returnedMetrics.nonEmpty) { + val addFiles = AddFileTags.partsMetricsToAddFile( + description.database, + description.tableName, + description.path, + returnedMetrics, + Seq(Utils.localHostName())) + + val (taskCommitMessage, taskCommitTime) = Utils.timeTakenMs { + // committer.commitTask(taskAttemptContext) + new TaskCommitMessage(addFiles.toSeq) + } + +// val summary = MergeTreeExecutedWriteSummary( +// updatedPartitions = updatedPartitions.toSet, +// stats = statsTrackers.map(_.getFinalStats(taskCommitTime))) + MergeTreeWriteTaskResult(taskCommitMessage, null) + } else { + throw new IllegalStateException() + } + })( + catchBlock = { + // If there is an error, abort the task + logError(s"Job $jobId aborted.") + }, + finallyBlock = {}) + } catch { + case e: FetchFailedException => + throw e + case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput => + // If any output file to write already exists, it does not make sense to re-run this task. + // We throw the exception and let Executor throw ExceptionFailure to abort the job. + throw new TaskOutputFileAlreadyExistException(f) + case t: Throwable => + throw QueryExecutionErrors.taskFailedWhileWritingRowsError(t) + } + + } + + def runOptimizeBinJobClickhouse( + txn: OptimisticTransaction, + partitionValues: Map[String, String], + bucketNum: String, + bin: Seq[AddFile], + maxFileSize: Long): Seq[FileAction] = { + val tableV2 = ClickHouseTableV2.getTable(txn.deltaLog); + + val sparkSession = SparkSession.getActiveSession.get + + val rddWithNonEmptyPartitions = + sparkSession.sparkContext.parallelize(Array.empty[InternalRow], 1) + + val jobIdInstant = new Date().getTime + val ret = new Array[MergeTreeWriteTaskResult](rddWithNonEmptyPartitions.partitions.length) + + val serializableHadoopConf = new SerializableConfiguration( + sparkSession.sessionState.newHadoopConfWithOptions( + txn.metadata.configuration ++ txn.deltaLog.options)) + + val partitionDir = if (tableV2.partitionColumns.isEmpty) { + None + } else { + Some(tableV2.partitionColumns.map(c => c + "=" + partitionValues(c)).mkString("/")) + } + + val bucketDir = if (tableV2.bucketOption.isEmpty) { + None + } else { + Some(bucketNum) + } + + val description = TaskDescription.apply( + txn.deltaLog.dataPath.toString, + tableV2.dataBaseName, + tableV2.tableName, + ClickhouseSnapshot.genSnapshotId(tableV2.snapshot), + tableV2.orderByKeyOption, + tableV2.lowCardKeyOption, + tableV2.minmaxIndexKeyOption, + tableV2.bfIndexKeyOption, + tableV2.setIndexKeyOption, + tableV2.primaryKeyOption, + tableV2.partitionColumns, + bin.map(_.asInstanceOf[AddMergeTreeParts].name), + tableV2.schema(), + tableV2.clickhouseTableConfigs, + serializableHadoopConf, + jobIdInstant, + partitionDir, + bucketDir + ) + sparkSession.sparkContext.runJob( + rddWithNonEmptyPartitions, + (taskContext: TaskContext, _: Iterator[InternalRow]) => { + executeTask( + description, + taskContext.stageId(), + taskContext.partitionId(), + taskContext.taskAttemptId().toInt & Integer.MAX_VALUE + ) + }, + rddWithNonEmptyPartitions.partitions.indices, + (index, res: MergeTreeWriteTaskResult) => { + ret(index) = res + } + ) + + val addFiles = ret + .flatMap(_.commitMsg.obj.asInstanceOf[Seq[AddFile]]) + .toSeq + + val removeFiles = + bin.map(f => f.removeWithTimestamp(new SystemClock().getTimeMillis(), dataChange = false)) + addFiles ++ removeFiles + + } + + def getDeltaLogClickhouse( + spark: SparkSession, + path: Option[String], + tableIdentifier: Option[TableIdentifier], + operationName: String, + hadoopConf: Map[String, String] = Map.empty): DeltaLog = { + val tablePath = + if (path.nonEmpty) { + new Path(path.get) + } else if (tableIdentifier.nonEmpty) { + val sessionCatalog = spark.sessionState.catalog + lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get) + + if (CHDataSourceUtils.isClickhousePath(spark, tableIdentifier.get)) { + new Path(tableIdentifier.get.table) + } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) { + new Path(metadata.location) + } else { + DeltaTableIdentifier(spark, tableIdentifier.get) match { + case Some(id) if id.path.nonEmpty => + new Path(id.path.get) + case Some(id) if id.table.nonEmpty => + new Path(metadata.location) + case _ => + if (metadata.tableType == CatalogTableType.VIEW) { + throw DeltaErrors.viewNotSupported(operationName) + } + throw DeltaErrors.notADeltaTableException(operationName) + } + } + } else { + throw DeltaErrors.missingTableIdentifierException(operationName) + } + + val startTime = Some(System.currentTimeMillis) + val deltaLog = DeltaLog.forTable(spark, tablePath, hadoopConf) + if (deltaLog.update(checkIfUpdatedSinceTs = startTime).version < 0) { + throw DeltaErrors.notADeltaTableException( + operationName, + DeltaTableIdentifier(path, tableIdentifier)) + } + deltaLog + } + + def groupFilesIntoBinsClickhouse( + partitionsToCompact: Seq[((String, Map[String, String]), Seq[AddFile])], + maxTargetFileSize: Long): Seq[((String, Map[String, String]), Seq[AddFile])] = { + partitionsToCompact.flatMap { + case (partition, files) => + val bins = new ArrayBuffer[Seq[AddFile]]() + + val currentBin = new ArrayBuffer[AddFile]() + var currentBinSize = 0L + + files.sortBy(_.size).foreach { + file => + // Generally, a bin is a group of existing files, whose total size does not exceed the + // desired maxFileSize. They will be coalesced into a single output file. + // However, if isMultiDimClustering = true, all files in a partition will be read by the + // same job, the data will be range-partitioned and + // numFiles = totalFileSize / maxFileSize + // will be produced. See below. + + // isMultiDimClustering is always false for Gluten Clickhouse for now + if (file.size + currentBinSize > maxTargetFileSize /* && !isMultiDimClustering */ ) { + bins += currentBin.toVector + currentBin.clear() + currentBin += file + currentBinSize = file.size + } else { + currentBin += file + currentBinSize += file.size + } + } + + if (currentBin.nonEmpty) { + bins += currentBin.toVector + } + + bins + .map(b => (partition, b)) + // select bins that have at least two files or in case of multi-dim clustering + // select all bins + .filter(_._2.size > 1 /* || isMultiDimClustering */ ) + } + } +} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala new file mode 100644 index 000000000000..8c1062f4c7b6 --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2.clickhouse + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.commands.cdc.CDCReader +import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils, DeltaSQLConf} +import org.apache.spark.sql.sources.BaseRelation +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +import org.apache.hadoop.fs.Path + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** A DataSource V1 for integrating Delta into Spark SQL batch and Streaming APIs. */ +class ClickHouseDataSource extends DeltaDataSource { + + override def shortName(): String = { + ClickHouseConfig.NAME + } + + override def getTable( + schema: StructType, + partitioning: Array[Transform], + properties: java.util.Map[String, String]): Table = { + val options = new CaseInsensitiveStringMap(properties) + val path = options.get("path") + if (path == null) throw DeltaErrors.pathNotSpecifiedException + new ClickHouseTableV2( + SparkSession.active, + new Path(path), + options = properties.asScala.toMap, + clickhouseExtensionOptions = ClickHouseConfig + .createMergeTreeConfigurations( + ClickHouseConfig + .getMergeTreeConfigurations(properties) + .asJava) + ) + } + + override def createRelation( + sqlContext: SQLContext, + mode: SaveMode, + parameters: Map[String, String], + data: DataFrame): BaseRelation = { + val path = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) + val partitionColumns = parameters + .get(DeltaSourceUtils.PARTITIONING_COLUMNS_KEY) + .map(DeltaDataSource.decodePartitioningColumns) + .getOrElse(Nil) + + val deltaLog = DeltaLog.forTable(sqlContext.sparkSession, path, parameters) + // need to use the latest snapshot + val configs = if (deltaLog.update().version < 0) { + // when creating table, save the clickhouse config to the delta metadata + val clickHouseTableV2 = ClickHouseTableV2.getTable(deltaLog) + clickHouseTableV2.properties().asScala.toMap ++ DeltaConfigs + .validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) + } else { + DeltaConfigs.validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) + } + WriteIntoDelta( + deltaLog = deltaLog, + mode = mode, + new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf), + partitionColumns = partitionColumns, + configuration = configs, + data = data + ).run(sqlContext.sparkSession) + + deltaLog.createRelation() + } + + override def createRelation( + sqlContext: SQLContext, + parameters: Map[String, String]): BaseRelation = { + recordFrameProfile("Delta", "DeltaDataSource.createRelation") { + val maybePath = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) + + // Log any invalid options that are being passed in + DeltaOptions.verifyOptions(CaseInsensitiveMap(parameters)) + + val timeTravelByParams = DeltaDataSource.getTimeTravelVersion(parameters) + var cdcOptions: mutable.Map[String, String] = mutable.Map.empty + val caseInsensitiveParams = new CaseInsensitiveStringMap(parameters.asJava) + if (CDCReader.isCDCRead(caseInsensitiveParams)) { + cdcOptions = mutable.Map[String, String](DeltaDataSource.CDC_ENABLED_KEY -> "true") + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_VERSION_KEY)) { + cdcOptions(DeltaDataSource.CDC_START_VERSION_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_START_VERSION_KEY) + } + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY)) { + cdcOptions(DeltaDataSource.CDC_START_TIMESTAMP_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_START_TIMESTAMP_KEY) + } + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_VERSION_KEY)) { + cdcOptions(DeltaDataSource.CDC_END_VERSION_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_END_VERSION_KEY) + } + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_TIMESTAMP_KEY)) { + cdcOptions(DeltaDataSource.CDC_END_TIMESTAMP_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_END_TIMESTAMP_KEY) + } + } + val dfOptions: Map[String, String] = + if ( + sqlContext.sparkSession.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) + ) { + parameters + } else { + Map.empty + } + (new ClickHouseTableV2( + sqlContext.sparkSession, + new Path(maybePath), + timeTravelOpt = timeTravelByParams, + options = dfOptions, + cdcOptions = new CaseInsensitiveStringMap(cdcOptions.asJava) + )).toBaseRelation + } + } +} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala new file mode 100644 index 000000000000..61e1da44d0af --- /dev/null +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala @@ -0,0 +1,662 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2.clickhouse + +import org.apache.gluten.sql.shims.SparkShimLoader + +import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException} +import org.apache.spark.sql.catalyst.catalog._ +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.connector.catalog._ +import org.apache.spark.sql.connector.catalog.TableCapability.V1_BATCH_WRITE +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} +import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError +import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} +import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} +import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.sources.InsertableRelation +import org.apache.spark.sql.types.StructType + +import org.apache.hadoop.fs.Path + +import java.util +import java.util.Locale + +import scala.collection.JavaConverters._ + +class ClickHouseSparkCatalog + extends DelegatingCatalogExtension + with StagingTableCatalog + with SupportsPathIdentifier + with DeltaLogging { + + val spark = SparkSession.active + + private def createCatalogTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String] + ): Table = { + super.createTable(ident, schema, partitions, properties) + } + + override def createTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): Table = { + if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + createClickHouseTable( + ident, + schema, + partitions, + properties, + Map.empty, + sourceQuery = None, + TableCreationModes.Create) + } else if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { + createDeltaTable( + ident, + schema, + partitions, + properties, + Map.empty, + sourceQuery = None, + TableCreationModes.Create + ) + } else { + createCatalogTable(ident, schema, partitions, properties) + } + } + + /** + * Creates a ClickHouse table + * + * @param ident + * The identifier of the table + * @param schema + * The schema of the table + * @param partitions + * The partition transforms for the table + * @param allTableProperties + * The table properties that configure the behavior of the table or provide information about + * the table + * @param writeOptions + * Options specific to the write during table creation or replacement + * @param sourceQuery + * A query if this CREATE request came from a CTAS or RTAS + * @param operation + * The specific table creation mode, whether this is a Create/Replace/Create or Replace + */ + private def createClickHouseTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + allTableProperties: util.Map[String, String], + writeOptions: Map[String, String], + sourceQuery: Option[DataFrame], + operation: TableCreationModes.CreationMode): Table = { + val (partitionColumns, maybeBucketSpec) = + SparkShimLoader.getSparkShims.convertPartitionTransforms(partitions) + var newSchema = schema + var newPartitionColumns = partitionColumns + var newBucketSpec = maybeBucketSpec + + // Delta does not support bucket feature, so save the bucket infos into properties if exists. + val tableProperties = + ClickHouseConfig.createMergeTreeConfigurations(allTableProperties, newBucketSpec) + + val isByPath = isPathIdentifier(ident) + val location = if (isByPath) { + Option(ident.name()) + } else { + Option(allTableProperties.get("location")) + } + val locUriOpt = location.map(CatalogUtils.stringToURI) + val storage = DataSource + .buildStorageFormatFromOptions(writeOptions) + .copy(locationUri = locUriOpt) + val tableType = + if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED + val id = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + val existingTableOpt = getExistingTableIfExists(id) + val loc = new Path(locUriOpt.getOrElse(spark.sessionState.catalog.defaultTablePath(id))) + val commentOpt = Option(allTableProperties.get("comment")) + + val tableDesc = new CatalogTable( + identifier = id, + tableType = tableType, + storage = storage, + schema = newSchema, + provider = Some(ClickHouseConfig.ALT_NAME), + partitionColumnNames = newPartitionColumns, + bucketSpec = newBucketSpec, + properties = tableProperties, + comment = commentOpt + ) + + val withDb = verifyTableAndSolidify(tableDesc, None, true) + + val writer = sourceQuery.map { + df => + WriteIntoDelta( + DeltaLog.forTable(spark, loc), + operation.mode, + new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), + withDb.partitionColumnNames, + withDb.properties ++ commentOpt.map("comment" -> _), + df, + schemaInCatalog = if (newSchema != schema) Some(newSchema) else None + ) + } + try { + ClickHouseTableV2.temporalThreadLocalCHTable.set( + new TempClickHouseTableV2(spark, Some(withDb))) + + CreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation = operation, + tableByPath = isByPath).run(spark) + } finally { + ClickHouseTableV2.temporalThreadLocalCHTable.remove() + } + + logInfo(s"create table ${ident.toString} successfully.") + loadTable(ident) + } + + /** + * Creates a Delta table + * + * @param ident + * The identifier of the table + * @param schema + * The schema of the table + * @param partitions + * The partition transforms for the table + * @param allTableProperties + * The table properties that configure the behavior of the table or provide information about + * the table + * @param writeOptions + * Options specific to the write during table creation or replacement + * @param sourceQuery + * A query if this CREATE request came from a CTAS or RTAS + * @param operation + * The specific table creation mode, whether this is a Create/Replace/Create or Replace + */ + private def createDeltaTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + allTableProperties: util.Map[String, String], + writeOptions: Map[String, String], + sourceQuery: Option[DataFrame], + operation: TableCreationModes.CreationMode + ): Table = { + // These two keys are tableProperties in data source v2 but not in v1, so we have to filter + // them out. Otherwise property consistency checks will fail. + val tableProperties = allTableProperties.asScala.filterKeys { + case TableCatalog.PROP_LOCATION => false + case TableCatalog.PROP_PROVIDER => false + case TableCatalog.PROP_COMMENT => false + case TableCatalog.PROP_OWNER => false + case TableCatalog.PROP_EXTERNAL => false + case "path" => false + case _ => true + }.toMap + val (partitionColumns, maybeBucketSpec) = + SparkShimLoader.getSparkShims.convertPartitionTransforms(partitions) + var newSchema = schema + var newPartitionColumns = partitionColumns + var newBucketSpec = maybeBucketSpec + val conf = spark.sessionState.conf + + val isByPath = isPathIdentifier(ident) + if ( + isByPath && !conf.getConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS) + && allTableProperties.containsKey("location") + // The location property can be qualified and different from the path in the identifier, so + // we check `endsWith` here. + && Option(allTableProperties.get("location")).exists(!_.endsWith(ident.name())) + ) { + throw DeltaErrors.ambiguousPathsInCreateTableException( + ident.name(), + allTableProperties.get("location")) + } + val location = if (isByPath) { + Option(ident.name()) + } else { + Option(allTableProperties.get("location")) + } + val id = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + var locUriOpt = location.map(CatalogUtils.stringToURI) + val existingTableOpt = getExistingTableIfExists(id) + val loc = locUriOpt + .orElse(existingTableOpt.flatMap(_.storage.locationUri)) + .getOrElse(spark.sessionState.catalog.defaultTablePath(id)) + val storage = DataSource + .buildStorageFormatFromOptions(writeOptions) + .copy(locationUri = Option(loc)) + val tableType = + if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED + val commentOpt = Option(allTableProperties.get("comment")) + + var tableDesc = new CatalogTable( + identifier = id, + tableType = tableType, + storage = storage, + schema = newSchema, + provider = Some(DeltaSourceUtils.ALT_NAME), + partitionColumnNames = newPartitionColumns, + bucketSpec = newBucketSpec, + properties = tableProperties, + comment = commentOpt + ) + + val withDb = verifyTableAndSolidify(tableDesc, None) + + val writer = sourceQuery.map { + df => + WriteIntoDelta( + DeltaLog.forTable(spark, new Path(loc)), + operation.mode, + new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), + withDb.partitionColumnNames, + withDb.properties ++ commentOpt.map("comment" -> _), + df, + schemaInCatalog = if (newSchema != schema) Some(newSchema) else None + ) + } + + CreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation, + tableByPath = isByPath).run(spark) + + loadTable(ident) + } + + /** Performs checks on the parameters provided for table creation for a ClickHouse table. */ + private def verifyTableAndSolidify( + tableDesc: CatalogTable, + query: Option[LogicalPlan], + isMergeTree: Boolean = false): CatalogTable = { + + if (!isMergeTree && tableDesc.bucketSpec.isDefined) { + throw DeltaErrors.operationNotSupportedException("Bucketing", tableDesc.identifier) + } + + val schema = query + .map { + plan => + assert(tableDesc.schema.isEmpty, "Can't specify table schema in CTAS.") + plan.schema.asNullable + } + .getOrElse(tableDesc.schema) + + PartitioningUtils.validatePartitionColumn( + schema, + tableDesc.partitionColumnNames, + caseSensitive = false + ) // Delta is case insensitive + + val validatedConfigurations = if (isMergeTree) { + tableDesc.properties + } else { + DeltaConfigs.validateConfigurations(tableDesc.properties) + } + + val db = tableDesc.identifier.database.getOrElse(catalog.getCurrentDatabase) + val tableIdentWithDB = tableDesc.identifier.copy(database = Some(db)) + tableDesc.copy( + identifier = tableIdentWithDB, + schema = schema, + properties = validatedConfigurations) + } + + /** Checks if a table already exists for the provided identifier. */ + def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { + // If this is a path identifier, we cannot return an existing CatalogTable. The Create command + // will check the file system itself + if (isPathIdentifier(table)) return None + val tableExists = catalog.tableExists(table) + if (tableExists) { + val oldTable = catalog.getTableMetadata(table) + if (oldTable.tableType == CatalogTableType.VIEW) { + throw new AnalysisException(s"$table is a view. You may not write data into a view.") + } + if ( + !DeltaSourceUtils.isDeltaTable(oldTable.provider) && + !CHDataSourceUtils.isClickHouseTable(oldTable.provider) + ) { + throw DeltaErrors.notADeltaTable(table.table) + } + Some(oldTable) + } else { + None + } + } + + private def getProvider(properties: util.Map[String, String]): String = { + Option(properties.get("provider")).getOrElse(ClickHouseConfig.NAME) + } + + override def loadTable(ident: Identifier): Table = { + try { + super.loadTable(ident) match { + case v1: V1Table if CHDataSourceUtils.isClickHouseTable(v1.catalogTable) => + new ClickHouseTableV2( + spark, + new Path(v1.catalogTable.location), + catalogTable = Some(v1.catalogTable), + tableIdentifier = Some(ident.toString)) + case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => + DeltaTableV2( + spark, + new Path(v1.catalogTable.location), + catalogTable = Some(v1.catalogTable), + tableIdentifier = Some(ident.toString)) + case o => + o + } + } catch { + case _: NoSuchDatabaseException | _: NoSuchNamespaceException | _: NoSuchTableException + if isPathIdentifier(ident) => + newDeltaPathTable(ident) + case e: AnalysisException if gluePermissionError(e) && isPathIdentifier(ident) => + logWarning( + "Received an access denied error from Glue. Assuming this " + + s"identifier ($ident) is path based.", + e) + newDeltaPathTable(ident) + } + } + + private def newDeltaPathTable(ident: Identifier): DeltaTableV2 = { + if (hasClickHouseNamespace(ident)) { + new ClickHouseTableV2(spark, new Path(ident.name())) + } else { + DeltaTableV2(spark, new Path(ident.name())) + } + } + + /** support to delete mergetree data from the external table */ + override def purgeTable(ident: Identifier): Boolean = { + try { + loadTable(ident) match { + case t: ClickHouseTableV2 => + val tableType = t.properties().getOrDefault("Type", "") + // file-based or external table + val isExternal = tableType.isEmpty || tableType.equalsIgnoreCase("external") + val tablePath = t.rootPath + // first delete the table metadata + val deletedTable = super.dropTable(ident) + if (deletedTable && isExternal) { + val fs = tablePath.getFileSystem(spark.sessionState.newHadoopConf()) + // delete all data if there is a external table + fs.delete(tablePath, true) + } + true + case _ => super.purgeTable(ident) + } + } catch { + case _: Exception => + false + } + } + + override def stageReplace( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): StagedTable = + recordFrameProfile("DeltaCatalog", "stageReplace") { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { + new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Replace) + } else { + super.dropTable(ident) + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) + } + } + + override def stageCreateOrReplace( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): StagedTable = + recordFrameProfile("DeltaCatalog", "stageCreateOrReplace") { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { + new StagedDeltaTableV2( + ident, + schema, + partitions, + properties, + TableCreationModes.CreateOrReplace) + } else { + try super.dropTable(ident) + catch { + case _: NoSuchDatabaseException => // this is fine + case _: NoSuchTableException => // this is fine + } + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) + } + } + + override def stageCreate( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): StagedTable = + recordFrameProfile("DeltaCatalog", "stageCreate") { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { + new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Create) + } else { + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) + } + } + + /** + * A staged delta table, which creates a HiveMetaStore entry and appends data if this was a + * CTAS/RTAS command. We have a ugly way of using this API right now, but it's the best way to + * maintain old behavior compatibility between Databricks Runtime and OSS Delta Lake. + */ + private class StagedDeltaTableV2( + ident: Identifier, + override val schema: StructType, + val partitions: Array[Transform], + override val properties: util.Map[String, String], + operation: TableCreationModes.CreationMode) + extends StagedTable + with SupportsWrite { + + private var asSelectQuery: Option[DataFrame] = None + private var writeOptions: Map[String, String] = Map.empty + + override def commitStagedChanges(): Unit = + recordFrameProfile("DeltaCatalog", "commitStagedChanges") { + val conf = spark.sessionState.conf + val props = new util.HashMap[String, String]() + // Options passed in through the SQL API will show up both with an "option." prefix and + // without in Spark 3.1, so we need to remove those from the properties + val optionsThroughProperties = properties.asScala.collect { + case (k, _) if k.startsWith("option.") => k.stripPrefix("option.") + }.toSet + val sqlWriteOptions = new util.HashMap[String, String]() + properties.asScala.foreach { + case (k, v) => + if (!k.startsWith("option.") && !optionsThroughProperties.contains(k)) { + // Do not add to properties + props.put(k, v) + } else if (optionsThroughProperties.contains(k)) { + sqlWriteOptions.put(k, v) + } + } + if (writeOptions.isEmpty && !sqlWriteOptions.isEmpty) { + writeOptions = sqlWriteOptions.asScala.toMap + } + if (conf.getConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS)) { + // Legacy behavior + writeOptions.foreach { case (k, v) => props.put(k, v) } + } else { + writeOptions.foreach { + case (k, v) => + // Continue putting in Delta prefixed options to avoid breaking workloads + if (k.toLowerCase(Locale.ROOT).startsWith("delta.")) { + props.put(k, v) + } + } + } + if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + createClickHouseTable( + ident, + schema, + partitions, + props, + writeOptions, + asSelectQuery, + operation) + } else { + createDeltaTable(ident, schema, partitions, props, writeOptions, asSelectQuery, operation) + } + } + + override def name(): String = ident.name() + + override def abortStagedChanges(): Unit = {} + + override def capabilities(): util.Set[TableCapability] = Set(V1_BATCH_WRITE).asJava + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + writeOptions = info.options.asCaseSensitiveMap().asScala.toMap + new DeltaV1WriteBuilder + } + + /* + * WriteBuilder for creating a Delta table. + */ + private class DeltaV1WriteBuilder extends WriteBuilder { + override def build(): V1Write = new V1Write { + override def toInsertableRelation(): InsertableRelation = { + new InsertableRelation { + override def insert(data: DataFrame, overwrite: Boolean): Unit = { + asSelectQuery = Option(data) + } + } + } + } + } + } + + private case class BestEffortStagedTable(ident: Identifier, table: Table, catalog: TableCatalog) + extends StagedTable + with SupportsWrite { + override def abortStagedChanges(): Unit = catalog.dropTable(ident) + + override def commitStagedChanges(): Unit = {} + + // Pass through + override def name(): String = table.name() + override def schema(): StructType = table.schema() + override def partitioning(): Array[Transform] = table.partitioning() + override def capabilities(): util.Set[TableCapability] = table.capabilities() + override def properties(): util.Map[String, String] = table.properties() + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { + case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) + case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) + } + } +} + +/** + * A trait for handling table access through clickhouse.`/some/path`. This is a stop-gap solution + * until PathIdentifiers are implemented in Apache Spark. + */ +trait SupportsPathIdentifier extends TableCatalog { + self: ClickHouseSparkCatalog => + + protected lazy val catalog: SessionCatalog = spark.sessionState.catalog + + override def tableExists(ident: Identifier): Boolean = { + if (isPathIdentifier(ident)) { + val path = new Path(ident.name()) + val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) + fs.exists(path) && fs.listStatus(path).nonEmpty + } else { + super.tableExists(ident) + } + } + + protected def isPathIdentifier(ident: Identifier): Boolean = { + // Should be a simple check of a special PathIdentifier class in the future + try { + supportSQLOnFile && (hasClickHouseNamespace(ident) || hasDeltaNamespace(ident)) && + new Path(ident.name()).isAbsolute + } catch { + case _: IllegalArgumentException => false + } + } + + protected def isPathIdentifier(table: CatalogTable): Boolean = { + isPathIdentifier(table.identifier) + } + + protected def isPathIdentifier(tableIdentifier: TableIdentifier): Boolean = { + isPathIdentifier(Identifier.of(tableIdentifier.database.toArray, tableIdentifier.table)) + } + + private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile + + protected def hasClickHouseNamespace(ident: Identifier): Boolean = { + ident.namespace().length == 1 && + CHDataSourceUtils.isClickHouseDataSourceName(ident.namespace().head) + } + + protected def hasDeltaNamespace(ident: Identifier): Boolean = { + ident.namespace().length == 1 && DeltaSourceUtils.isDeltaDataSourceName(ident.namespace().head) + } +} diff --git a/backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala new file mode 100644 index 000000000000..790b4c1f8a37 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.tables + +import org.apache.spark.sql.{Dataset, Row, SparkSession} +import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils} +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 + +import org.apache.hadoop.fs.Path + +import scala.collection.JavaConverters._ + +class ClickhouseTable( + @transient private val _df: Dataset[Row], + @transient private val table: ClickHouseTableV2) + extends DeltaTable(_df, table) { + + override def optimize(): DeltaOptimizeBuilder = { + DeltaOptimizeBuilder(table) + } +} + +object ClickhouseTable { + + /** + * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path + * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a + * `not a Delta table` error. + * + * Note: This uses the active SparkSession in the current thread to read the table data. Hence, + * this throws error if active SparkSession has not been set, that is, + * `SparkSession.getActiveSession()` is empty. + * + * @since 0.3.0 + */ + def forPath(path: String): DeltaTable = { + val sparkSession = SparkSession.getActiveSession.getOrElse { + throw DeltaErrors.activeSparkSessionNotFound() + } + forPath(sparkSession, path) + } + + /** + * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path + * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a + * `not a Delta table` error. + * + * @since 0.3.0 + */ + def forPath(sparkSession: SparkSession, path: String): DeltaTable = { + forPath(sparkSession, path, Map.empty[String, String]) + } + + /** + * Instantiate a [[DeltaTable]] object representing the data at the given path, If the given path + * is invalid (i.e. either no table exists or an existing table is not a Delta table), it throws a + * `not a Delta table` error. + * + * @param hadoopConf + * Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to + * access the file system when executing queries. Other configurations will not be allowed. + * + * {{{ + * val hadoopConf = Map( + * "fs.s3a.access.key" -> "", + * "fs.s3a.secret.key" -> "" + * ) + * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) + * }}} + * @since 2.2.0 + */ + def forPath( + sparkSession: SparkSession, + path: String, + hadoopConf: scala.collection.Map[String, String]): DeltaTable = { + // We only pass hadoopConf so that we won't pass any unsafe options to Delta. + val badOptions = hadoopConf.filterKeys { + k => !DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) + }.toMap + if (!badOptions.isEmpty) { + throw DeltaErrors.unsupportedDeltaTableForPathHadoopConf(badOptions) + } + val fileSystemOptions: Map[String, String] = hadoopConf.toMap + val hdpPath = new Path(path) + if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath, fileSystemOptions)) { + new ClickhouseTable( + sparkSession.read.format("clickhouse").options(fileSystemOptions).load(path), + new ClickHouseTableV2(spark = sparkSession, path = hdpPath, options = fileSystemOptions) + ) + } else { + throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path))) + } + } + + /** + * Java friendly API to instantiate a [[DeltaTable]] object representing the data at the given + * path, If the given path is invalid (i.e. either no table exists or an existing table is not a + * Delta table), it throws a `not a Delta table` error. + * + * @param hadoopConf + * Hadoop configuration starting with "fs." or "dfs." will be picked up by `DeltaTable` to + * access the file system when executing queries. Other configurations will be ignored. + * + * {{{ + * val hadoopConf = Map( + * "fs.s3a.access.key" -> "", + * "fs.s3a.secret.key", "" + * ) + * DeltaTable.forPath(spark, "/path/to/table", hadoopConf) + * }}} + * @since 2.2.0 + */ + def forPath( + sparkSession: SparkSession, + path: String, + hadoopConf: java.util.Map[String, String]): DeltaTable = { + val fsOptions = hadoopConf.asScala.toMap + forPath(sparkSession, path, fsOptions) + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala new file mode 100644 index 000000000000..d8ab2c1d078c --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta + +import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.execution.ColumnarToRowExecBase + +import org.apache.spark.SparkException +import org.apache.spark.sql.Dataset +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 +import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} +import org.apache.spark.sql.delta.files.MergeTreeCommitProtocol +import org.apache.spark.sql.delta.schema.InvariantViolationException +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec +import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker} +import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.util.SerializableConfiguration + +import org.apache.commons.lang3.exception.ExceptionUtils + +import scala.collection.mutable.ListBuffer + +object ClickhouseOptimisticTransaction {} + +class ClickhouseOptimisticTransaction( + override val deltaLog: DeltaLog, + override val catalogTable: Option[CatalogTable], + override val snapshot: Snapshot) + extends OptimisticTransaction(deltaLog, catalogTable, snapshot) { + + def this( + deltaLog: DeltaLog, + catalogTable: Option[CatalogTable], + snapshotOpt: Option[Snapshot] = None) { + this( + deltaLog, + catalogTable, + snapshotOpt.getOrElse(deltaLog.update()) + ) + } + + def insertFakeRowAdaptor(queryPlan: SparkPlan): SparkPlan = queryPlan match { + // if the child is columnar, we can just wrap&transfer the columnar data + case c2r: ColumnarToRowExecBase => + FakeRowAdaptor(c2r.child) + // If the child is aqe, we make aqe "support columnar", + // then aqe itself will guarantee to generate columnar outputs. + // So FakeRowAdaptor will always consumes columnar data, + // thus avoiding the case of c2r->aqe->r2c->writer + case aqe: AdaptiveSparkPlanExec => + FakeRowAdaptor( + AdaptiveSparkPlanExec( + aqe.inputPlan, + aqe.context, + aqe.preprocessingRules, + aqe.isSubquery, + supportsColumnar = true + )) + case other => FakeRowAdaptor(other) + } + + override def writeFiles( + inputData: Dataset[_], + writeOptions: Option[DeltaOptions], + additionalConstraints: Seq[Constraint]): Seq[FileAction] = { + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + hasWritten = true + + val spark = inputData.sparkSession + val (data, partitionSchema) = performCDCPartition(inputData) + val outputPath = deltaLog.dataPath + + val (queryExecution, output, generatedColumnConstraints, _) = + normalizeData(deltaLog, writeOptions, data) + val partitioningColumns = getPartitioningColumns(partitionSchema, output) + + val committer = + new MergeTreeCommitProtocol("delta-mergetree", outputPath.toString, None, None) + + // val (optionalStatsTracker, _) = + // getOptionalStatsTrackerAndStatsCollection(output, outputPath, partitionSchema, data) + val (optionalStatsTracker, _) = (None, None) + + val constraints = + Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints + + SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { + val outputSpec = FileFormatWriter.OutputSpec(outputPath.toString, Map.empty, output) + + val queryPlan = queryExecution.executedPlan + val newQueryPlan = insertFakeRowAdaptor(queryPlan) + + val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() + + if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { + val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( + new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), + BasicWriteJobStatsTracker.metrics) + // registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) + statsTrackers.append(basicWriteJobStatsTracker) + } + + // Iceberg spec requires partition columns in data files + val writePartitionColumns = IcebergCompat.isAnyEnabled(metadata) + // Retain only a minimal selection of Spark writer options to avoid any potential + // compatibility issues + var options = (writeOptions match { + case None => Map.empty[String, String] + case Some(writeOptions) => + writeOptions.options.filterKeys { + key => + key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || + key.equalsIgnoreCase(DeltaOptions.COMPRESSION) + }.toMap + }) + (DeltaOptions.WRITE_PARTITION_COLUMNS -> writePartitionColumns.toString) + + spark.conf.getAll.foreach( + entry => { + if ( + entry._1.startsWith(s"${CHBackendSettings.getBackendConfigPrefix}.runtime_settings") + || entry._1.equalsIgnoreCase(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key) + ) { + options += (entry._1 -> entry._2) + } + }) + + try { + val tableV2 = ClickHouseTableV2.getTable(deltaLog) + MergeTreeFileFormatWriter.write( + sparkSession = spark, + plan = newQueryPlan, + fileFormat = tableV2.getFileFormat(protocol, metadata), + // formats. + committer = committer, + outputSpec = outputSpec, + // scalastyle:off deltahadoopconfiguration + hadoopConf = spark.sessionState + .newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), + // scalastyle:on deltahadoopconfiguration + orderByKeyOption = tableV2.orderByKeyOption, + lowCardKeyOption = tableV2.lowCardKeyOption, + minmaxIndexKeyOption = tableV2.minmaxIndexKeyOption, + bfIndexKeyOption = tableV2.bfIndexKeyOption, + setIndexKeyOption = tableV2.setIndexKeyOption, + primaryKeyOption = tableV2.primaryKeyOption, + partitionColumns = partitioningColumns, + bucketSpec = tableV2.bucketOption, + statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers, + options = options, + constraints = constraints + ) + } catch { + case s: SparkException => + // Pull an InvariantViolationException up to the top level if it was the root cause. + val violationException = ExceptionUtils.getRootCause(s) + if (violationException.isInstanceOf[InvariantViolationException]) { + throw violationException + } else { + throw s + } + } + } + committer.addedStatuses.toSeq ++ committer.changeFiles + } else { + // TODO: support native delta parquet write + // 1. insert FakeRowAdaptor + // 2. DeltaInvariantCheckerExec transform + // 3. DeltaTaskStatisticsTracker collect null count / min values / max values + // 4. set the parameters 'staticPartitionWriteOnly', 'isNativeAppliable', + // 'nativeFormat' in the LocalProperty of the sparkcontext + super.writeFiles(inputData, writeOptions, additionalConstraints) + } + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala new file mode 100644 index 000000000000..8a9c5585e888 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta + +object DeltaAdapter extends DeltaAdapterTrait { + override def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.unsafeVolatileSnapshot +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala new file mode 100644 index 000000000000..dca14d7fb1fb --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala @@ -0,0 +1,1106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +// scalastyle:off import.ordering.noEmptyLine +import java.io.IOException +import java.lang.ref.WeakReference +import java.net.URI +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.control.NonFatal + +import com.databricks.spark.util.TagDefinitions._ +import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeLogFileIndex} +import org.apache.spark.sql.delta.managedcommit.ManagedCommitUtils +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} +import org.apache.spark.sql.delta.sources._ +import org.apache.spark.sql.delta.storage.LogStoreProvider +import org.apache.spark.sql.delta.util.FileNames +import com.google.common.cache.{Cache, CacheBuilder, RemovalNotification} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.{FileSourceOptions, TableIdentifier} +import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} +import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expression, Literal} +import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper +import org.apache.spark.sql.catalyst.util.FailFastMode +import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.expressions.UserDefinedFunction +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} +import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.util._ + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0, it is modified to overcome the following issues: + * 1. return ClickhouseOptimisticTransaction + * 2. return DeltaMergeTreeFileFormat + * 3. create HadoopFsRelation with the bucket options + */ + +/** + * Used to query the current state of the log as well as modify it by adding + * new atomic collections of actions. + * + * Internally, this class implements an optimistic concurrency control + * algorithm to handle multiple readers or writers. Any single read + * is guaranteed to see a consistent snapshot of the table. + * + * @param logPath Path of the Delta log JSONs. + * @param dataPath Path of the data files. + * @param options Filesystem options filtered from `allOptions`. + * @param allOptions All options provided by the user, for example via `df.write.option()`. This + * includes but not limited to filesystem and table properties. + * @param clock Clock to be used when starting a new transaction. + */ +class DeltaLog private( + val logPath: Path, + val dataPath: Path, + val options: Map[String, String], + val allOptions: Map[String, String], + val clock: Clock + ) extends Checkpoints + with MetadataCleanup + with LogStoreProvider + with SnapshotManagement + with DeltaFileFormat + with ProvidesUniFormConverters + with ReadChecksum { + + import org.apache.spark.sql.delta.files.TahoeFileIndex + + /** + * Path to sidecar directory. + * This is intentionally kept `lazy val` as otherwise any other constructor codepaths in DeltaLog + * (e.g. SnapshotManagement etc) will see it as null as they are executed before this line is + * called. + */ + lazy val sidecarDirPath: Path = FileNames.sidecarDirPath(logPath) + + + protected def spark = SparkSession.active + + checkRequiredConfigurations() + + /** + * Keep a reference to `SparkContext` used to create `DeltaLog`. `DeltaLog` cannot be used when + * `SparkContext` is stopped. We keep the reference so that we can check whether the cache is + * still valid and drop invalid `DeltaLog`` objects. + */ + private val sparkContext = new WeakReference(spark.sparkContext) + + /** + * Returns the Hadoop [[Configuration]] object which can be used to access the file system. All + * Delta code should use this method to create the Hadoop [[Configuration]] object, so that the + * hadoop file system configurations specified in DataFrame options will come into effect. + */ + // scalastyle:off deltahadoopconfiguration + final def newDeltaHadoopConf(): Configuration = + spark.sessionState.newHadoopConfWithOptions(options) + // scalastyle:on deltahadoopconfiguration + + /** Used to read and write physical log files and checkpoints. */ + lazy val store = createLogStore(spark) + + /** Delta History Manager containing version and commit history. */ + lazy val history = new DeltaHistoryManager( + this, spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) + + /* --------------- * + | Configuration | + * --------------- */ + + /** + * The max lineage length of a Snapshot before Delta forces to build a Snapshot from scratch. + * Delta will build a Snapshot on top of the previous one if it doesn't see a checkpoint. + * However, there is a race condition that when two writers are writing at the same time, + * a writer may fail to pick up checkpoints written by another one, and the lineage will grow + * and finally cause StackOverflowError. Hence we have to force to build a Snapshot from scratch + * when the lineage length is too large to avoid hitting StackOverflowError. + */ + def maxSnapshotLineageLength: Int = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH) + + /** The unique identifier for this table. */ + def tableId: String = unsafeVolatileMetadata.id // safe because table id never changes + + /** + * Combines the tableId with the path of the table to ensure uniqueness. Normally `tableId` + * should be globally unique, but nothing stops users from copying a Delta table directly to + * a separate location, where the transaction log is copied directly, causing the tableIds to + * match. When users mutate the copied table, and then try to perform some checks joining the + * two tables, optimizations that depend on `tableId` alone may not be correct. Hence we use a + * composite id. + */ + private[delta] def compositeId: (String, Path) = tableId -> dataPath + + /** + * Creates a [[LogicalRelation]] for a given [[DeltaLogFileIndex]], with all necessary file source + * options taken from the Delta Log. All reads of Delta metadata files should use this method. + */ + def indexToRelation( + index: DeltaLogFileIndex, + schema: StructType = Action.logSchema): LogicalRelation = { + DeltaLog.indexToRelation(spark, index, options, schema) + } + + /** + * Load the data using the FileIndex. This allows us to skip many checks that add overhead, e.g. + * file existence checks, partitioning schema inference. + */ + def loadIndex( + index: DeltaLogFileIndex, + schema: StructType = Action.logSchema): DataFrame = { + Dataset.ofRows(spark, indexToRelation(index, schema)) + } + + /* ------------------ * + | Delta Management | + * ------------------ */ + + /** + * Returns a new [[OptimisticTransaction]] that can be used to read the current state of the log + * and then commit updates. The reads and updates will be checked for logical conflicts with any + * concurrent writes to the log, and post-commit hooks can be used to notify the table's catalog + * of schema changes, etc. + * + * Note that all reads in a transaction must go through the returned transaction object, and not + * directly to the [[DeltaLog]] otherwise they will not be checked for conflicts. + * + * @param catalogTableOpt The [[CatalogTable]] for the table this transaction updates. Passing + * None asserts this is a path-based table with no catalog entry. + * + * @param snapshotOpt THe [[Snapshot]] this transaction should use, if not latest. + */ + def startTransaction( + catalogTableOpt: Option[CatalogTable], + snapshotOpt: Option[Snapshot] = None): OptimisticTransaction = { + TransactionExecutionObserver.threadObserver.get().startingTransaction { + // --- modified start + new ClickhouseOptimisticTransaction(this, catalogTableOpt, snapshotOpt) + // --- modified end + } + } + + /** Legacy/compat overload that does not require catalog table information. Avoid prod use. */ + // --- modified start + // @deprecated("Please use the CatalogTable overload instead", "3.0") + // --- modified end + def startTransaction(): OptimisticTransaction = { + startTransaction(catalogTableOpt = None, snapshotOpt = None) + } + + /** + * Execute a piece of code within a new [[OptimisticTransaction]]. Reads/write sets will + * be recorded for this table, and all other tables will be read + * at a snapshot that is pinned on the first access. + * + * @param catalogTableOpt The [[CatalogTable]] for the table this transaction updates. Passing + * None asserts this is a path-based table with no catalog entry. + * + * @param snapshotOpt THe [[Snapshot]] this transaction should use, if not latest. + * @note This uses thread-local variable to make the active transaction visible. So do not use + * multi-threaded code in the provided thunk. + */ + def withNewTransaction[T]( + catalogTableOpt: Option[CatalogTable], + snapshotOpt: Option[Snapshot] = None)( + thunk: OptimisticTransaction => T): T = { + try { + val txn = startTransaction(catalogTableOpt, snapshotOpt) + OptimisticTransaction.setActive(txn) + thunk(txn) + } finally { + OptimisticTransaction.clearActive() + } + } + + /** Legacy/compat overload that does not require catalog table information. Avoid prod use. */ + @deprecated("Please use the CatalogTable overload instead", "3.0") + def withNewTransaction[T](thunk: OptimisticTransaction => T): T = { + try { + val txn = startTransaction() + OptimisticTransaction.setActive(txn) + thunk(txn) + } finally { + OptimisticTransaction.clearActive() + } + } + + + /** + * Upgrade the table's protocol version, by default to the maximum recognized reader and writer + * versions in this Delta release. This method only upgrades protocol version, and will fail if + * the new protocol version is not a superset of the original one used by the snapshot. + */ + def upgradeProtocol( + catalogTable: Option[CatalogTable], + snapshot: Snapshot, + newVersion: Protocol): Unit = { + val currentVersion = snapshot.protocol + if (newVersion == currentVersion) { + logConsole(s"Table $dataPath is already at protocol version $newVersion.") + return + } + if (!currentVersion.canUpgradeTo(newVersion)) { + throw new ProtocolDowngradeException(currentVersion, newVersion) + } + + val txn = startTransaction(catalogTable, Some(snapshot)) + try { + SchemaMergingUtils.checkColumnNameDuplication(txn.metadata.schema, "in the table schema") + } catch { + case e: AnalysisException => + throw DeltaErrors.duplicateColumnsOnUpdateTable(e) + } + txn.commit(Seq(newVersion), DeltaOperations.UpgradeProtocol(newVersion)) + logConsole(s"Upgraded table at $dataPath to $newVersion.") + } + + /** + * Get all actions starting from "startVersion" (inclusive). If `startVersion` doesn't exist, + * return an empty Iterator. + * Callers are encouraged to use the other override which takes the endVersion if available to + * avoid I/O and improve performance of this method. + */ + def getChanges( + startVersion: Long, + failOnDataLoss: Boolean = false): Iterator[(Long, Seq[Action])] = { + getChangeLogFiles(startVersion, failOnDataLoss).map { case (version, status) => + (version, store.read(status, newDeltaHadoopConf()).map(Action.fromJson(_))) + } + } + + private[sql] def getChanges( + startVersion: Long, + endVersion: Long, + failOnDataLoss: Boolean): Iterator[(Long, Seq[Action])] = { + getChangeLogFiles(startVersion, endVersion, failOnDataLoss).map { case (version, status) => + (version, store.read(status, newDeltaHadoopConf()).map(Action.fromJson(_))) + } + } + + private[sql] def getChangeLogFiles( + startVersion: Long, + endVersion: Long, + failOnDataLoss: Boolean): Iterator[(Long, FileStatus)] = { + implicit class IteratorWithStopAtHelper[T](underlying: Iterator[T]) { + // This method is used to stop the iterator when the condition is met. + def stopAt(stopAtFunc: (T) => Boolean): Iterator[T] = new Iterator[T] { + var shouldStop = false + + override def hasNext: Boolean = !shouldStop && underlying.hasNext + + override def next(): T = { + val v = underlying.next() + shouldStop = stopAtFunc(v) + v + } + } + } + + getChangeLogFiles(startVersion, failOnDataLoss) + // takeWhile always looks at one extra item, which can trigger unnecessary work. Instead, we + // stop if we've seen the item we believe should be the last interesting item, without + // examining the one that follows. + .stopAt { case (version, _) => version >= endVersion } + // The last element in this iterator may not be <= endVersion, so we need to filter it out. + .takeWhile { case (version, _) => version <= endVersion } + } + + /** + * Get access to all actions starting from "startVersion" (inclusive) via [[FileStatus]]. + * If `startVersion` doesn't exist, return an empty Iterator. + * Callers are encouraged to use the other override which takes the endVersion if available to + * avoid I/O and improve performance of this method. + */ + def getChangeLogFiles( + startVersion: Long, + failOnDataLoss: Boolean = false): Iterator[(Long, FileStatus)] = { + val deltasWithVersion = ManagedCommitUtils.commitFilesIterator(this, startVersion) + // Subtract 1 to ensure that we have the same check for the inclusive startVersion + var lastSeenVersion = startVersion - 1 + deltasWithVersion.map { case (status, version) => + if (failOnDataLoss && version > lastSeenVersion + 1) { + throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) + } + lastSeenVersion = version + (version, status) + } + } + + /* --------------------- * + | Protocol validation | + * --------------------- */ + + /** + * Asserts the highest protocol supported by this client is not less than what required by the + * table for performing read or write operations. This ensures the client to support a + * greater-or-equal protocol versions and recognizes/supports all features enabled by the table. + * + * The operation type to be checked is passed as a string in `readOrWrite`. Valid values are + * `read` and `write`. + */ + private def protocolCheck(tableProtocol: Protocol, readOrWrite: String): Unit = { + val clientSupportedProtocol = Action.supportedProtocolVersion() + // Depending on the operation, pull related protocol versions out of Protocol objects. + // `getEnabledFeatures` is a pointer to pull reader/writer features out of a Protocol. + val (clientSupportedVersions, tableRequiredVersion, getEnabledFeatures) = readOrWrite match { + case "read" => ( + Action.supportedReaderVersionNumbers, + tableProtocol.minReaderVersion, + (f: Protocol) => f.readerFeatureNames) + case "write" => ( + Action.supportedWriterVersionNumbers, + tableProtocol.minWriterVersion, + (f: Protocol) => f.writerFeatureNames) + case _ => + throw new IllegalArgumentException("Table operation must be either `read` or `write`.") + } + + // Check is complete when both the protocol version and all referenced features are supported. + val clientSupportedFeatureNames = getEnabledFeatures(clientSupportedProtocol) + val tableEnabledFeatureNames = getEnabledFeatures(tableProtocol) + if (tableEnabledFeatureNames.subsetOf(clientSupportedFeatureNames) && + clientSupportedVersions.contains(tableRequiredVersion)) { + return + } + + // Otherwise, either the protocol version, or few features referenced by the table, is + // unsupported. + val clientUnsupportedFeatureNames = + tableEnabledFeatureNames.diff(clientSupportedFeatureNames) + // Prepare event log constants and the appropriate error message handler. + val (opType, versionKey, unsupportedFeaturesException) = readOrWrite match { + case "read" => ( + "delta.protocol.failure.read", + "minReaderVersion", + DeltaErrors.unsupportedReaderTableFeaturesInTableException _) + case "write" => ( + "delta.protocol.failure.write", + "minWriterVersion", + DeltaErrors.unsupportedWriterTableFeaturesInTableException _) + } + recordDeltaEvent( + this, + opType, + data = Map( + "clientVersion" -> clientSupportedVersions.max, + versionKey -> tableRequiredVersion, + "clientFeatures" -> clientSupportedFeatureNames.mkString(","), + "clientUnsupportedFeatures" -> clientUnsupportedFeatureNames.mkString(","))) + if (!clientSupportedVersions.contains(tableRequiredVersion)) { + throw new InvalidProtocolVersionException( + dataPath.toString(), + tableProtocol.minReaderVersion, + tableProtocol.minWriterVersion, + Action.supportedReaderVersionNumbers.toSeq, + Action.supportedWriterVersionNumbers.toSeq) + } else { + throw unsupportedFeaturesException(dataPath.toString(), clientUnsupportedFeatureNames) + } + } + + /** + * Asserts that the table's protocol enabled all features that are active in the metadata. + * + * A mismatch shouldn't happen when the table has gone through a proper write process because we + * require all active features during writes. However, other clients may void this guarantee. + */ + def assertTableFeaturesMatchMetadata( + targetProtocol: Protocol, + targetMetadata: Metadata): Unit = { + if (!targetProtocol.supportsReaderFeatures && !targetProtocol.supportsWriterFeatures) return + + val protocolEnabledFeatures = targetProtocol.writerFeatureNames + .flatMap(TableFeature.featureNameToFeature) + val activeFeatures = + Protocol.extractAutomaticallyEnabledFeatures(spark, targetMetadata, Some(targetProtocol)) + val activeButNotEnabled = activeFeatures.diff(protocolEnabledFeatures) + if (activeButNotEnabled.nonEmpty) { + throw DeltaErrors.tableFeatureMismatchException(activeButNotEnabled.map(_.name)) + } + } + + /** + * Asserts that the client is up to date with the protocol and allowed to read the table that is + * using the given `protocol`. + */ + def protocolRead(protocol: Protocol): Unit = { + protocolCheck(protocol, "read") + } + + /** + * Asserts that the client is up to date with the protocol and allowed to write to the table + * that is using the given `protocol`. + */ + def protocolWrite(protocol: Protocol): Unit = { + protocolCheck(protocol, "write") + } + + /* ---------------------------------------- * + | Log Directory Management and Retention | + * ---------------------------------------- */ + + /** + * Whether a Delta table exists at this directory. + * It is okay to use the cached volatile snapshot here, since the worst case is that the table + * has recently started existing which hasn't been picked up here. If so, any subsequent command + * that updates the table will see the right value. + */ + def tableExists: Boolean = unsafeVolatileSnapshot.version >= 0 + + def isSameLogAs(otherLog: DeltaLog): Boolean = this.compositeId == otherLog.compositeId + + /** Creates the log directory if it does not exist. */ + def ensureLogDirectoryExist(): Unit = { + val fs = logPath.getFileSystem(newDeltaHadoopConf()) + def createDirIfNotExists(path: Path): Unit = { + // Optimistically attempt to create the directory first without checking its existence. + // This is efficient because we're assuming it's more likely that the directory doesn't + // exist and it saves an filesystem existence check in that case. + val (success, mkdirsIOExceptionOpt) = try { + // Return value of false should mean the directory already existed (not an error) but + // we will verify below because we're paranoid about buggy FileSystem implementations. + (fs.mkdirs(path), None) + } catch { + // A FileAlreadyExistsException is expected if a non-directory object exists but an explicit + // check is needed because buggy Hadoop FileSystem.mkdir wrongly throws the exception even + // on existing directories. + case io: IOException => + val dirExists = + try { + fs.getFileStatus(path).isDirectory + } catch { + case NonFatal(_) => false + } + (dirExists, Some(io)) + } + if (!success) { + throw DeltaErrors.cannotCreateLogPathException( + logPath = logPath.toString, + cause = mkdirsIOExceptionOpt.orNull) + } + } + createDirIfNotExists(FileNames.commitDirPath(logPath)) + } + + /** + * Create the log directory. Unlike `ensureLogDirectoryExist`, this method doesn't check whether + * the log directory exists and it will ignore the return value of `mkdirs`. + */ + def createLogDirectory(): Unit = { + logPath.getFileSystem(newDeltaHadoopConf()).mkdirs(logPath) + } + + /* ------------ * + | Integration | + * ------------ */ + + /** + * Returns a [[org.apache.spark.sql.DataFrame]] containing the new files within the specified + * version range. + */ + def createDataFrame( + snapshot: SnapshotDescriptor, + addFiles: Seq[AddFile], + isStreaming: Boolean = false, + actionTypeOpt: Option[String] = None): DataFrame = { + val actionType = actionTypeOpt.getOrElse(if (isStreaming) "streaming" else "batch") + // It's ok to not pass down the partitionSchema to TahoeBatchFileIndex. Schema evolution will + // ensure any partitionSchema changes will be captured, and upon restart, the new snapshot will + // be initialized with the correct partition schema again. + val fileIndex = new TahoeBatchFileIndex(spark, actionType, addFiles, this, dataPath, snapshot) + // --- modified start + // TODO: Don't add the bucketOption here, it will cause the OOM when the merge into update + // key is the bucket column, fix later + // --- modified end + val relation = buildHadoopFsRelationWithFileIndex(snapshot, fileIndex, bucketSpec = None) + Dataset.ofRows(spark, LogicalRelation(relation, isStreaming = isStreaming)) + } + + /** + * Returns a [[BaseRelation]] that contains all of the data present + * in the table. This relation will be continually updated + * as files are added or removed from the table. However, new [[BaseRelation]] + * must be requested in order to see changes to the schema. + */ + def createRelation( + partitionFilters: Seq[Expression] = Nil, + snapshotToUseOpt: Option[Snapshot] = None, + catalogTableOpt: Option[CatalogTable] = None, + isTimeTravelQuery: Boolean = false): BaseRelation = { + + /** Used to link the files present in the table into the query planner. */ + // TODO: If snapshotToUse is unspecified, get the correct snapshot from update() + val snapshotToUse = snapshotToUseOpt.getOrElse(unsafeVolatileSnapshot) + if (snapshotToUse.version < 0) { + // A negative version here means the dataPath is an empty directory. Read query should error + // out in this case. + throw DeltaErrors.pathNotExistsException(dataPath.toString) + } + + val fileIndex = TahoeLogFileIndex( + spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) + // --- modified start + var bucketSpec: Option[BucketSpec] = + if (ClickHouseConfig.isMergeTreeFormatEngine(snapshotToUse.metadata.configuration)) { + ClickHouseTableV2.getTable(this).bucketOption + } else { + None + } + + val r = buildHadoopFsRelationWithFileIndex(snapshotToUse, fileIndex, bucketSpec = bucketSpec) + new DeltaLog.DeltaHadoopFsRelation( + r.location, + r.partitionSchema, + r.dataSchema, + r.bucketSpec, + r.fileFormat, + r.options + )(spark, this, catalogTableOpt) + // --- modified end + } + + def buildHadoopFsRelationWithFileIndex(snapshot: SnapshotDescriptor, fileIndex: TahoeFileIndex, + bucketSpec: Option[BucketSpec]): HadoopFsRelation = { + HadoopFsRelation( + fileIndex, + partitionSchema = DeltaColumnMapping.dropColumnMappingMetadata( + snapshot.metadata.partitionSchema), + // We pass all table columns as `dataSchema` so that Spark will preserve the partition + // column locations. Otherwise, for any partition columns not in `dataSchema`, Spark would + // just append them to the end of `dataSchema`. + dataSchema = DeltaColumnMapping.dropColumnMappingMetadata( + DeltaTableUtils.removeInternalMetadata(spark, + SchemaUtils.dropNullTypeColumns(snapshot.metadata.schema))), + bucketSpec = bucketSpec, + fileFormat(snapshot.protocol, snapshot.metadata), + // `metadata.format.options` is not set today. Even if we support it in future, we shouldn't + // store any file system options since they may contain credentials. Hence, it will never + // conflict with `DeltaLog.options`. + snapshot.metadata.format.options ++ options)(spark) + } + + /** + * Verify the required Spark conf for delta + * Throw `DeltaErrors.configureSparkSessionWithExtensionAndCatalog` exception if + * `spark.sql.catalog.spark_catalog` config is missing. We do not check for + * `spark.sql.extensions` because DeltaSparkSessionExtension can alternatively + * be activated using the `.withExtension()` API. This check can be disabled + * by setting DELTA_CHECK_REQUIRED_SPARK_CONF to false. + */ + protected def checkRequiredConfigurations(): Unit = { + if (spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_REQUIRED_SPARK_CONFS_CHECK)) { + if (spark.conf.getOption( + SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key).isEmpty) { + throw DeltaErrors.configureSparkSessionWithExtensionAndCatalog(None) + } + } + } + + /** + * Returns a proper path canonicalization function for the current Delta log. + * + * If `runsOnExecutors` is true, the returned method will use a broadcast Hadoop Configuration + * so that the method is suitable for execution on executors. Otherwise, the returned method + * will use a local Hadoop Configuration and the method can only be executed on the driver. + */ + private[delta] def getCanonicalPathFunction(runsOnExecutors: Boolean): String => String = { + val hadoopConf = newDeltaHadoopConf() + // Wrap `hadoopConf` with a method to delay the evaluation to run on executors. + val getHadoopConf = if (runsOnExecutors) { + val broadcastHadoopConf = + spark.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) + () => broadcastHadoopConf.value.value + } else { + () => hadoopConf + } + + new DeltaLog.CanonicalPathFunction(getHadoopConf) + } + + /** + * Returns a proper path canonicalization UDF for the current Delta log. + * + * If `runsOnExecutors` is true, the returned UDF will use a broadcast Hadoop Configuration. + * Otherwise, the returned UDF will use a local Hadoop Configuration and the UDF can + * only be executed on the driver. + */ + private[delta] def getCanonicalPathUdf(runsOnExecutors: Boolean = true): UserDefinedFunction = { + DeltaUDF.stringFromString(getCanonicalPathFunction(runsOnExecutors)) + } + + // --- modified start + override def fileFormat(protocol: Protocol, metadata: Metadata): FileFormat = { + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + ClickHouseTableV2.getTable(this).getFileFormat(protocol, metadata) + } else { + super.fileFormat(protocol, metadata) + } + } + // --- modified end +} + +object DeltaLog extends DeltaLogging { + + // --- modified start + @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) + private class DeltaHadoopFsRelation( + location: FileIndex, + partitionSchema: StructType, + // The top-level columns in `dataSchema` should match the actual physical file schema, + // otherwise the ORC data source may not work with the by-ordinal mode. + dataSchema: StructType, + bucketSpec: Option[BucketSpec], + fileFormat: FileFormat, + options: Map[String, String] + )(spark: SparkSession, deltaLog: DeltaLog, catalogTableOpt: Option[CatalogTable]) + extends HadoopFsRelation( + location, + partitionSchema, + dataSchema, + bucketSpec, + fileFormat, + options)(spark) + with InsertableRelation { + def insert(data: DataFrame, overwrite: Boolean): Unit = { + val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append + WriteIntoDelta( + deltaLog = deltaLog, + mode = mode, + new DeltaOptions(Map.empty[String, String], spark.sessionState.conf), + partitionColumns = Seq.empty, + configuration = Map.empty, + data = data, + catalogTableOpt = catalogTableOpt).run(spark) + } + } + // --- modified end + + /** + * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file + * system options (options starting with "fs." or "dfs." prefix) passed into + * `DataFrameReader/Writer` + */ + private type DeltaLogCacheKey = (Path, Map[String, String]) + + /** The name of the subdirectory that holds Delta metadata files */ + private[delta] val LOG_DIR_NAME = "_delta_log" + + private[delta] def logPathFor(dataPath: String): Path = logPathFor(new Path(dataPath)) + private[delta] def logPathFor(dataPath: Path): Path = + DeltaTableUtils.safeConcatPaths(dataPath, LOG_DIR_NAME) + + /** + * We create only a single [[DeltaLog]] for any given `DeltaLogCacheKey` to avoid wasted work + * in reconstructing the log. + */ + type CacheKey = (Path, Map[String, String]) + private[delta] def getOrCreateCache(conf: SQLConf): + Cache[CacheKey, DeltaLog] = synchronized { + deltaLogCache match { + case Some(c) => c + case None => + val builder = createCacheBuilder(conf) + .removalListener( + (removalNotification: RemovalNotification[DeltaLogCacheKey, DeltaLog]) => { + val log = removalNotification.getValue + // TODO: We should use ref-counting to uncache snapshots instead of a manual timed op + try log.unsafeVolatileSnapshot.uncache() catch { + case _: java.lang.NullPointerException => + // Various layers will throw null pointer if the RDD is already gone. + } + }) + deltaLogCache = Some(builder.build[CacheKey, DeltaLog]()) + deltaLogCache.get + } + } + + private var deltaLogCache: Option[Cache[CacheKey, DeltaLog]] = None + + /** + * Helper to create delta log caches + */ + private def createCacheBuilder(conf: SQLConf): CacheBuilder[AnyRef, AnyRef] = { + val cacheRetention = conf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES) + val cacheSize = conf + .getConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE) + .max(sys.props.get("delta.log.cacheSize").map(_.toLong).getOrElse(0L)) + + CacheBuilder + .newBuilder() + .expireAfterAccess(cacheRetention, TimeUnit.MINUTES) + .maximumSize(cacheSize) + } + + + /** + * Creates a [[LogicalRelation]] for a given [[DeltaLogFileIndex]], with all necessary file source + * options taken from the Delta Log. All reads of Delta metadata files should use this method. + */ + def indexToRelation( + spark: SparkSession, + index: DeltaLogFileIndex, + additionalOptions: Map[String, String], + schema: StructType = Action.logSchema): LogicalRelation = { + val formatSpecificOptions: Map[String, String] = index.format match { + case DeltaLogFileIndex.COMMIT_FILE_FORMAT => + jsonCommitParseOption + case _ => Map.empty + } + // Delta should NEVER ignore missing or corrupt metadata files, because doing so can render the + // entire table unusable. Hard-wire that into the file source options so the user can't override + // it by setting spark.sql.files.ignoreCorruptFiles or spark.sql.files.ignoreMissingFiles. + val allOptions = additionalOptions ++ formatSpecificOptions ++ Map( + FileSourceOptions.IGNORE_CORRUPT_FILES -> "false", + FileSourceOptions.IGNORE_MISSING_FILES -> "false" + ) + // --- modified start + // Don't need to add the bucketOption here, it handles the delta log meta json file + // --- modified end + val fsRelation = HadoopFsRelation( + index, index.partitionSchema, schema, None, index.format, allOptions)(spark) + LogicalRelation(fsRelation) + } + + // Don't tolerate malformed JSON when parsing Delta log actions (default is PERMISSIVE) + val jsonCommitParseOption = Map("mode" -> FailFastMode.name) + + /** Helper for creating a log when it stored at the root of the data. */ + def forTable(spark: SparkSession, dataPath: String): DeltaLog = { + apply(spark, logPathFor(dataPath), Map.empty, new SystemClock) + } + + /** Helper for creating a log when it stored at the root of the data. */ + def forTable(spark: SparkSession, dataPath: Path): DeltaLog = { + apply(spark, logPathFor(dataPath), new SystemClock) + } + + /** Helper for creating a log when it stored at the root of the data. */ + def forTable(spark: SparkSession, dataPath: Path, options: Map[String, String]): DeltaLog = { + apply(spark, logPathFor(dataPath), options, new SystemClock) + } + + /** Helper for creating a log when it stored at the root of the data. */ + def forTable(spark: SparkSession, dataPath: Path, clock: Clock): DeltaLog = { + apply(spark, logPathFor(dataPath), clock) + } + + /** Helper for creating a log for the table. */ + def forTable(spark: SparkSession, tableName: TableIdentifier): DeltaLog = { + forTable(spark, tableName, new SystemClock) + } + + /** Helper for creating a log for the table. */ + def forTable(spark: SparkSession, table: CatalogTable): DeltaLog = { + forTable(spark, table, new SystemClock) + } + + /** Helper for creating a log for the table. */ + def forTable(spark: SparkSession, tableName: TableIdentifier, clock: Clock): DeltaLog = { + if (DeltaTableIdentifier.isDeltaPath(spark, tableName)) { + forTable(spark, new Path(tableName.table)) + } else { + forTable(spark, spark.sessionState.catalog.getTableMetadata(tableName), clock) + } + } + + /** Helper for creating a log for the table. */ + def forTable(spark: SparkSession, table: CatalogTable, clock: Clock): DeltaLog = { + apply(spark, logPathFor(new Path(table.location)), clock) + } + + private def apply(spark: SparkSession, rawPath: Path, clock: Clock = new SystemClock): DeltaLog = + apply(spark, rawPath, Map.empty, clock) + + + /** Helper for getting a log, as well as the latest snapshot, of the table */ + def forTableWithSnapshot(spark: SparkSession, dataPath: String): (DeltaLog, Snapshot) = + withFreshSnapshot { forTable(spark, new Path(dataPath), _) } + + /** Helper for getting a log, as well as the latest snapshot, of the table */ + def forTableWithSnapshot(spark: SparkSession, dataPath: Path): (DeltaLog, Snapshot) = + withFreshSnapshot { forTable(spark, dataPath, _) } + + /** Helper for getting a log, as well as the latest snapshot, of the table */ + def forTableWithSnapshot( + spark: SparkSession, + tableName: TableIdentifier): (DeltaLog, Snapshot) = + withFreshSnapshot { forTable(spark, tableName, _) } + + /** Helper for getting a log, as well as the latest snapshot, of the table */ + def forTableWithSnapshot( + spark: SparkSession, + dataPath: Path, + options: Map[String, String]): (DeltaLog, Snapshot) = + withFreshSnapshot { apply(spark, logPathFor(dataPath), options, _) } + + /** + * Helper function to be used with the forTableWithSnapshot calls. Thunk is a + * partially applied DeltaLog.forTable call, which we can then wrap around with a + * snapshot update. We use the system clock to avoid back-to-back updates. + */ + private[delta] def withFreshSnapshot(thunk: Clock => DeltaLog): (DeltaLog, Snapshot) = { + val clock = new SystemClock + val ts = clock.getTimeMillis() + val deltaLog = thunk(clock) + val snapshot = deltaLog.update(checkIfUpdatedSinceTs = Some(ts)) + (deltaLog, snapshot) + } + + private def apply( + spark: SparkSession, + rawPath: Path, + options: Map[String, String], + clock: Clock + ): DeltaLog = { + val fileSystemOptions: Map[String, String] = + if (spark.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { + // We pick up only file system options so that we don't pass any parquet or json options to + // the code that reads Delta transaction logs. + options.filterKeys { k => + DeltaTableUtils.validDeltaTableHadoopPrefixes.exists(k.startsWith) + }.toMap + } else { + Map.empty + } + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) + // scalastyle:on deltahadoopconfiguration + val fs = rawPath.getFileSystem(hadoopConf) + val path = fs.makeQualified(rawPath) + def createDeltaLog(): DeltaLog = recordDeltaOperation( + null, + "delta.log.create", + Map(TAG_TAHOE_PATH -> path.getParent.toString)) { + AnalysisHelper.allowInvokingTransformsInAnalyzer { + new DeltaLog( + logPath = path, + dataPath = path.getParent, + options = fileSystemOptions, + allOptions = options, + clock = clock + ) + } + } + def getDeltaLogFromCache(): DeltaLog = { + // The following cases will still create a new ActionLog even if there is a cached + // ActionLog using a different format path: + // - Different `scheme` + // - Different `authority` (e.g., different user tokens in the path) + // - Different mount point. + try { + getOrCreateCache(spark.sessionState.conf) + .get(path -> fileSystemOptions, () => { + createDeltaLog() + } + ) + } catch { + case e: com.google.common.util.concurrent.UncheckedExecutionException => throw e.getCause + case e: java.util.concurrent.ExecutionException => throw e.getCause + } + } + + val deltaLog = getDeltaLogFromCache() + if (Option(deltaLog.sparkContext.get).map(_.isStopped).getOrElse(true)) { + // Invalid the cached `DeltaLog` and create a new one because the `SparkContext` of the cached + // `DeltaLog` has been stopped. + getOrCreateCache(spark.sessionState.conf).invalidate(path -> fileSystemOptions) + getDeltaLogFromCache() + } else { + deltaLog + } + } + + /** Invalidate the cached DeltaLog object for the given `dataPath`. */ + def invalidateCache(spark: SparkSession, dataPath: Path): Unit = { + try { + val rawPath = logPathFor(dataPath) + // scalastyle:off deltahadoopconfiguration + // This method cannot be called from DataFrameReader/Writer so it's safe to assume the user + // has set the correct file system configurations in the session configs. + val fs = rawPath.getFileSystem(spark.sessionState.newHadoopConf()) + // scalastyle:on deltahadoopconfiguration + val path = fs.makeQualified(rawPath) + + val deltaLogCache = getOrCreateCache(spark.sessionState.conf) + if (spark.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS)) { + // We rely on the fact that accessing the key set doesn't modify the entry access time. See + // `CacheBuilder.expireAfterAccess`. + val keysToBeRemoved = mutable.ArrayBuffer[DeltaLogCacheKey]() + val iter = deltaLogCache.asMap().keySet().iterator() + while (iter.hasNext) { + val key = iter.next() + if (key._1 == path) { + keysToBeRemoved += key + } + } + deltaLogCache.invalidateAll(keysToBeRemoved.asJava) + } else { + deltaLogCache.invalidate(path -> Map.empty) + } + } catch { + case NonFatal(e) => logWarning(e.getMessage, e) + } + } + + def clearCache(): Unit = { + deltaLogCache.foreach(_.invalidateAll()) + } + + /** Unset the caches. Exposing for testing */ + private[delta] def unsetCache(): Unit = { + synchronized { + deltaLogCache = None + } + } + + /** Return the number of cached `DeltaLog`s. Exposing for testing */ + private[delta] def cacheSize: Long = { + deltaLogCache.map(_.size()).getOrElse(0L) + } + + /** + * Filters the given [[Dataset]] by the given `partitionFilters`, returning those that match. + * @param files The active files in the DeltaLog state, which contains the partition value + * information + * @param partitionFilters Filters on the partition columns + * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested + * @param shouldRewritePartitionFilters Whether to rewrite `partitionFilters` to be over the + * [[AddFile]] schema + */ + def filterFileList( + partitionSchema: StructType, + files: DataFrame, + partitionFilters: Seq[Expression], + partitionColumnPrefixes: Seq[String] = Nil, + shouldRewritePartitionFilters: Boolean = true): DataFrame = { + + val rewrittenFilters = if (shouldRewritePartitionFilters) { + rewritePartitionFilters( + partitionSchema, + files.sparkSession.sessionState.conf.resolver, + partitionFilters, + partitionColumnPrefixes) + } else { + partitionFilters + } + val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) + val columnFilter = new Column(expr) + files.filter(columnFilter) + } + + /** + * Rewrite the given `partitionFilters` to be used for filtering partition values. + * We need to explicitly resolve the partitioning columns here because the partition columns + * are stored as keys of a Map type instead of attributes in the AddFile schema (below) and thus + * cannot be resolved automatically. + * + * @param partitionFilters Filters on the partition columns + * @param partitionColumnPrefixes The path to the `partitionValues` column, if it's nested + */ + def rewritePartitionFilters( + partitionSchema: StructType, + resolver: Resolver, + partitionFilters: Seq[Expression], + partitionColumnPrefixes: Seq[String] = Nil): Seq[Expression] = { + partitionFilters + .map(_.transformUp { + case a: Attribute => + // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns + // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. + val unquoted = a.name.stripPrefix("`").stripSuffix("`") + val partitionCol = partitionSchema.find { field => resolver(field.name, unquoted) } + partitionCol match { + case Some(f: StructField) => + val name = DeltaColumnMapping.getPhysicalName(f) + Cast( + UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", name)), + f.dataType) + case None => + // This should not be able to happen, but the case was present in the original code so + // we kept it to be safe. + log.error(s"Partition filter referenced column ${a.name} not in the partition schema") + UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", a.name)) + } + }) + } + + + /** + * Checks whether this table only accepts appends. If so it will throw an error in operations that + * can remove data such as DELETE/UPDATE/MERGE. + */ + def assertRemovable(snapshot: Snapshot): Unit = { + val metadata = snapshot.metadata + if (DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata)) { + throw DeltaErrors.modifyAppendOnlyTableException(metadata.name) + } + } + + /** How long to keep around SetTransaction actions before physically deleting them. */ + def minSetTransactionRetentionInterval(metadata: Metadata): Option[Long] = { + DeltaConfigs.TRANSACTION_ID_RETENTION_DURATION + .fromMetaData(metadata) + .map(DeltaConfigs.getMilliSeconds) + } + /** How long to keep around logically deleted files before physically deleting them. */ + def tombstoneRetentionMillis(metadata: Metadata): Long = { + DeltaConfigs.getMilliSeconds(DeltaConfigs.TOMBSTONE_RETENTION.fromMetaData(metadata)) + } + + /** Get a function that canonicalizes a given `path`. */ + private[delta] class CanonicalPathFunction(getHadoopConf: () => Configuration) + extends Function[String, String] with Serializable { + // Mark it `@transient lazy val` so that de-serialization happens only once on every executor. + @transient + private lazy val fs = { + // scalastyle:off FileSystemGet + FileSystem.get(getHadoopConf()) + // scalastyle:on FileSystemGet + } + + override def apply(path: String): String = { + // scalastyle:off pathfromuri + val hadoopPath = new Path(new URI(path)) + // scalastyle:on pathfromuri + if (hadoopPath.isAbsoluteAndSchemeAuthorityNull) { + fs.makeQualified(hadoopPath).toUri.toString + } else { + // return untouched if it is a relative path or is already fully qualified + hadoopPath.toUri.toString + } + } + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala new file mode 100644 index 000000000000..13a91f051dae --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala @@ -0,0 +1,663 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +// scalastyle:off import.ordering.noEmptyLine +import scala.collection.mutable + +import org.apache.spark.sql.delta.actions._ +import org.apache.spark.sql.delta.actions.Action.logSchema +import org.apache.spark.sql.delta.managedcommit.{CommitOwnerProvider, TableCommitOwnerClient} +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.schema.SchemaUtils +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.DataSkippingReader +import org.apache.spark.sql.delta.stats.DeltaScan +import org.apache.spark.sql.delta.stats.DeltaStatsColumnSpec +import org.apache.spark.sql.delta.stats.StatisticsCollection +import org.apache.spark.sql.delta.util.DeltaCommitFileProvider +import org.apache.spark.sql.delta.util.FileNames +import org.apache.spark.sql.delta.util.StateCache +import org.apache.spark.sql.util.ScalaExtensions._ +import org.apache.hadoop.fs.{FileStatus, Path} + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.StructType +import org.apache.spark.util.Utils + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0. It is modified to overcome the following issues: + * 1. filesForScan() will cache the DeltaScan by the FilterExprsAsKey + * 2. filesForScan() should return DeltaScan of AddMergeTreeParts instead of AddFile + */ + +/** + * A description of a Delta [[Snapshot]], including basic information such its [[DeltaLog]] + * metadata, protocol, and version. + */ +trait SnapshotDescriptor { + def deltaLog: DeltaLog + def version: Long + def metadata: Metadata + def protocol: Protocol + + def schema: StructType = metadata.schema + + protected[delta] def numOfFilesIfKnown: Option[Long] + protected[delta] def sizeInBytesIfKnown: Option[Long] +} + +/** + * An immutable snapshot of the state of the log at some delta version. Internally + * this class manages the replay of actions stored in checkpoint or delta files. + * + * After resolving any new actions, it caches the result and collects the + * following basic information to the driver: + * - Protocol Version + * - Metadata + * - Transaction state + * + * @param inCommitTimestampOpt The in-commit-timestamp of the latest commit in milliseconds. Can + * be set to None if + * 1. The timestamp has not been read yet - generally the case for cold tables. + * 2. Or the table has not been initialized, i.e. `version = -1`. + * 3. Or the table does not have [[InCommitTimestampTableFeature]] enabled. + * + */ +class Snapshot( + val path: Path, + override val version: Long, + val logSegment: LogSegment, + override val deltaLog: DeltaLog, + val checksumOpt: Option[VersionChecksum] + ) + extends SnapshotDescriptor + with SnapshotStateManager + with StateCache + with StatisticsCollection + with DataSkippingReader + with DeltaLogging { + + import Snapshot._ + import DeltaLogFileIndex.COMMIT_VERSION_COLUMN + // For implicits which re-use Encoder: + import org.apache.spark.sql.delta.implicits._ + + protected def spark = SparkSession.active + + /** Snapshot to scan by the DeltaScanGenerator for metadata query optimizations */ + override val snapshotToScan: Snapshot = this + + override def columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode + + /** + * Returns the timestamp of the latest commit of this snapshot. + * For an uninitialized snapshot, this returns -1. + * + * When InCommitTimestampTableFeature is enabled, the timestamp + * is retrieved from the CommitInfo of the latest commit which + * can result in an IO operation. + */ + def timestamp: Long = + getInCommitTimestampOpt.getOrElse(logSegment.lastCommitFileModificationTimestamp) + + /** + * Returns the inCommitTimestamp if ICT is enabled, otherwise returns None. + * This potentially triggers an IO operation to read the inCommitTimestamp. + * This is a lazy val, so repeated calls will not trigger multiple IO operations. + */ + protected lazy val getInCommitTimestampOpt: Option[Long] = + Option.when(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata)) { + _reconstructedProtocolMetadataAndICT.inCommitTimestamp + .getOrElse { + val startTime = System.currentTimeMillis() + var exception = Option.empty[Throwable] + try { + val commitInfoOpt = DeltaHistoryManager.getCommitInfoOpt( + deltaLog.store, + DeltaCommitFileProvider(this).deltaFile(version), + deltaLog.newDeltaHadoopConf()) + CommitInfo.getRequiredInCommitTimestamp(commitInfoOpt, version.toString) + } catch { + case e: Throwable => + exception = Some(e) + throw e + } finally { + recordDeltaEvent( + deltaLog, + "delta.inCommitTimestamp.read", + data = Map( + "version" -> version, + "callSite" -> "Snapshot.getInCommitTimestampOpt", + "checkpointVersion" -> logSegment.checkpointProvider.version, + "durationMs" -> (System.currentTimeMillis() - startTime), + "exceptionMessage" -> exception.map(_.getMessage).getOrElse(""), + "exceptionStackTrace" -> exception.map(_.getStackTrace.mkString("\n")).getOrElse("") + ) + ) + } + } + } + + + private[delta] lazy val nonFileActions: Seq[Action] = { + Seq(protocol, metadata) ++ + setTransactions ++ + domainMetadata + } + + @volatile private[delta] var stateReconstructionTriggered = false + + /** + * Use [[stateReconstruction]] to create a representation of the actions in this table. + * Cache the resultant output. + */ + private lazy val cachedState = recordFrameProfile("Delta", "snapshot.cachedState") { + stateReconstructionTriggered = true + cacheDS(stateReconstruction, s"Delta Table State #$version - $redactedPath") + } + + /** + * Given the list of files from `LogSegment`, create respective file indices to help create + * a DataFrame and short-circuit the many file existence and partition schema inference checks + * that exist in DataSource.resolveRelation(). + */ + protected[delta] lazy val deltaFileIndexOpt: Option[DeltaLogFileIndex] = { + assertLogFilesBelongToTable(path, logSegment.deltas) + DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, logSegment.deltas) + } + + protected lazy val fileIndices: Seq[DeltaLogFileIndex] = { + val checkpointFileIndexes = checkpointProvider.allActionsFileIndexes() + checkpointFileIndexes ++ deltaFileIndexOpt.toSeq + } + + /** + * Protocol, Metadata, and In-Commit Timestamp retrieved through + * `protocolMetadataAndICTReconstruction` which skips a full state reconstruction. + */ + case class ReconstructedProtocolMetadataAndICT( + protocol: Protocol, + metadata: Metadata, + inCommitTimestamp: Option[Long]) + + /** + * Generate the protocol and metadata for this snapshot. This is usually cheaper than a + * full state reconstruction, but still only compute it when necessary. + */ + private lazy val _reconstructedProtocolMetadataAndICT: ReconstructedProtocolMetadataAndICT = + { + // Should be small. At most 'checkpointInterval' rows, unless new commits are coming + // in before a checkpoint can be written + var protocol: Protocol = null + var metadata: Metadata = null + var inCommitTimestamp: Option[Long] = None + protocolMetadataAndICTReconstruction().foreach { + case ReconstructedProtocolMetadataAndICT(p: Protocol, _, _) => protocol = p + case ReconstructedProtocolMetadataAndICT(_, m: Metadata, _) => metadata = m + case ReconstructedProtocolMetadataAndICT(_, _, ict: Option[Long]) => inCommitTimestamp = ict + } + + if (protocol == null) { + recordDeltaEvent( + deltaLog, + opType = "delta.assertions.missingAction", + data = Map( + "version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) + throw DeltaErrors.actionNotFoundException("protocol", version) + } + + if (metadata == null) { + recordDeltaEvent( + deltaLog, + opType = "delta.assertions.missingAction", + data = Map( + "version" -> version.toString, "action" -> "Metadata", "source" -> "Snapshot")) + throw DeltaErrors.actionNotFoundException("metadata", version) + } + + ReconstructedProtocolMetadataAndICT(protocol, metadata, inCommitTimestamp) + } + + /** + * [[CommitOwnerClient]] for the given delta table as of this snapshot. + * - This must be present when managed commit is enabled. + * - This must be None when managed commit is disabled. + */ + val tableCommitOwnerClientOpt: Option[TableCommitOwnerClient] = initializeTableCommitOwner() + protected def initializeTableCommitOwner(): Option[TableCommitOwnerClient] = { + CommitOwnerProvider.getTableCommitOwner(this) + } + + /** Number of columns to collect stats on for data skipping */ + override lazy val statsColumnSpec: DeltaStatsColumnSpec = + StatisticsCollection.configuredDeltaStatsColumnSpec(metadata) + + /** Performs validations during initialization */ + protected def init(): Unit = { + deltaLog.protocolRead(protocol) + deltaLog.assertTableFeaturesMatchMetadata(protocol, metadata) + SchemaUtils.recordUndefinedTypes(deltaLog, metadata.schema) + } + + /** The current set of actions in this [[Snapshot]] as plain Rows */ + def stateDF: DataFrame = recordFrameProfile("Delta", "stateDF") { + cachedState.getDF + } + + /** The current set of actions in this [[Snapshot]] as a typed Dataset. */ + def stateDS: Dataset[SingleAction] = recordFrameProfile("Delta", "stateDS") { + cachedState.getDS + } + + private[delta] def allFilesViaStateReconstruction: Dataset[AddFile] = { + stateDS.where("add IS NOT NULL").select(col("add").as[AddFile]) + } + + // Here we need to bypass the ACL checks for SELECT anonymous function permissions. + /** All of the files present in this [[Snapshot]]. */ + def allFiles: Dataset[AddFile] = allFilesViaStateReconstruction + + /** All unexpired tombstones. */ + def tombstones: Dataset[RemoveFile] = { + stateDS.where("remove IS NOT NULL").select(col("remove").as[RemoveFile]) + } + + def deltaFileSizeInBytes(): Long = deltaFileIndexOpt.map(_.sizeInBytes).getOrElse(0L) + + def checkpointSizeInBytes(): Long = checkpointProvider.effectiveCheckpointSizeInBytes() + + override def metadata: Metadata = _reconstructedProtocolMetadataAndICT.metadata + + override def protocol: Protocol = _reconstructedProtocolMetadataAndICT.protocol + + /** + * Pulls the protocol and metadata of the table from the files that are used to compute the + * Snapshot directly--without triggering a full state reconstruction. This is important, because + * state reconstruction depends on protocol and metadata for correctness. + * If the current table version does not have a checkpoint, this function will also return the + * in-commit-timestamp of the latest commit if available. + * + * Also this method should only access methods defined in [[UninitializedCheckpointProvider]] + * which are not present in [[CheckpointProvider]]. This is because initialization of + * [[Snapshot.checkpointProvider]] depends on [[Snapshot.protocolMetadataAndICTReconstruction()]] + * and so if [[Snapshot.protocolMetadataAndICTReconstruction()]] starts depending on + * [[Snapshot.checkpointProvider]] then there will be cyclic dependency. + */ + protected def protocolMetadataAndICTReconstruction(): + Array[ReconstructedProtocolMetadataAndICT] = { + import implicits._ + + val schemaToUse = Action.logSchema(Set("protocol", "metaData", "commitInfo")) + val checkpointOpt = checkpointProvider.topLevelFileIndex.map { index => + deltaLog.loadIndex(index, schemaToUse) + .withColumn(COMMIT_VERSION_COLUMN, lit(checkpointProvider.version)) + } + (checkpointOpt ++ deltaFileIndexOpt.map(deltaLog.loadIndex(_, schemaToUse)).toSeq) + .reduceOption(_.union(_)).getOrElse(emptyDF) + .select("protocol", "metaData", "commitInfo.inCommitTimestamp", COMMIT_VERSION_COLUMN) + .where("protocol.minReaderVersion is not null or metaData.id is not null " + + s"or (commitInfo.inCommitTimestamp is not null and version = $version)") + .as[(Protocol, Metadata, Option[Long], Long)] + .collect() + .sortBy(_._4) + .map { + case (p, m, ict, _) => ReconstructedProtocolMetadataAndICT(p, m, ict) + } + } + + // Reconstruct the state by applying deltas in order to the checkpoint. + // We partition by path as it is likely the bulk of the data is add/remove. + // Non-path based actions will be collocated to a single partition. + protected def stateReconstruction: Dataset[SingleAction] = { + recordFrameProfile("Delta", "snapshot.stateReconstruction") { + // for serializability + val localMinFileRetentionTimestamp = minFileRetentionTimestamp + val localMinSetTransactionRetentionTimestamp = minSetTransactionRetentionTimestamp + + val canonicalPath = deltaLog.getCanonicalPathUdf() + + // Canonicalize the paths so we can repartition the actions correctly, but only rewrite the + // add/remove actions themselves after partitioning and sorting are complete. Otherwise, the + // optimizer can generate a really bad plan that re-evaluates _EVERY_ field of the rewritten + // struct(...) projection every time we touch _ANY_ field of the rewritten struct. + // + // NOTE: We sort by [[COMMIT_VERSION_COLUMN]] (provided by [[loadActions]]), to ensure that + // actions are presented to InMemoryLogReplay in the ascending version order it expects. + val ADD_PATH_CANONICAL_COL_NAME = "add_path_canonical" + val REMOVE_PATH_CANONICAL_COL_NAME = "remove_path_canonical" + loadActions + .withColumn(ADD_PATH_CANONICAL_COL_NAME, when( + col("add.path").isNotNull, canonicalPath(col("add.path")))) + .withColumn(REMOVE_PATH_CANONICAL_COL_NAME, when( + col("remove.path").isNotNull, canonicalPath(col("remove.path")))) + .repartition( + getNumPartitions, + coalesce(col(ADD_PATH_CANONICAL_COL_NAME), col(REMOVE_PATH_CANONICAL_COL_NAME))) + .sortWithinPartitions(COMMIT_VERSION_COLUMN) + .withColumn("add", when( + col("add.path").isNotNull, + struct( + col(ADD_PATH_CANONICAL_COL_NAME).as("path"), + col("add.partitionValues"), + col("add.size"), + col("add.modificationTime"), + col("add.dataChange"), + col(ADD_STATS_TO_USE_COL_NAME).as("stats"), + col("add.tags"), + col("add.deletionVector"), + col("add.baseRowId"), + col("add.defaultRowCommitVersion"), + col("add.clusteringProvider") + ))) + .withColumn("remove", when( + col("remove.path").isNotNull, + col("remove").withField("path", col(REMOVE_PATH_CANONICAL_COL_NAME)))) + .as[SingleAction] + .mapPartitions { iter => + val state: LogReplay = + new InMemoryLogReplay( + localMinFileRetentionTimestamp, + localMinSetTransactionRetentionTimestamp) + state.append(0, iter.map(_.unwrap)) + state.checkpoint.map(_.wrap) + } + } + } + + /** + * Loads the file indices into a DataFrame that can be used for LogReplay. + * + * In addition to the usual nested columns provided by the SingleAction schema, it should provide + * two additional columns to simplify the log replay process: [[COMMIT_VERSION_COLUMN]] (which, + * when sorted in ascending order, will order older actions before newer ones, as required by + * [[InMemoryLogReplay]]); and [[ADD_STATS_TO_USE_COL_NAME]] (to handle certain combinations of + * config settings for delta.checkpoint.writeStatsAsJson and delta.checkpoint.writeStatsAsStruct). + */ + protected def loadActions: DataFrame = { + fileIndices.map(deltaLog.loadIndex(_)) + .reduceOption(_.union(_)).getOrElse(emptyDF) + .withColumn(ADD_STATS_TO_USE_COL_NAME, col("add.stats")) + } + + /** + * Tombstones before the [[minFileRetentionTimestamp]] timestamp will be dropped from the + * checkpoint. + */ + private[delta] def minFileRetentionTimestamp: Long = { + deltaLog.clock.getTimeMillis() - DeltaLog.tombstoneRetentionMillis(metadata) + } + + /** + * [[SetTransaction]]s before [[minSetTransactionRetentionTimestamp]] will be considered expired + * and dropped from the snapshot. + */ + private[delta] def minSetTransactionRetentionTimestamp: Option[Long] = { + DeltaLog.minSetTransactionRetentionInterval(metadata).map(deltaLog.clock.getTimeMillis() - _) + } + + private[delta] def getNumPartitions: Int = { + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS) + .getOrElse(Snapshot.defaultNumSnapshotPartitions) + } + + /** + * Computes all the information that is needed by the checksum for the current snapshot. + * May kick off state reconstruction if needed by any of the underlying fields. + * Note that it's safe to set txnId to none, since the snapshot doesn't always have a txn + * attached. E.g. if a snapshot is created by reading a checkpoint, then no txnId is present. + */ + def computeChecksum: VersionChecksum = VersionChecksum( + txnId = None, + tableSizeBytes = sizeInBytes, + numFiles = numOfFiles, + numMetadata = numOfMetadata, + numProtocol = numOfProtocol, + inCommitTimestampOpt = getInCommitTimestampOpt, + setTransactions = checksumOpt.flatMap(_.setTransactions), + domainMetadata = domainMetadatasIfKnown, + metadata = metadata, + protocol = protocol, + histogramOpt = fileSizeHistogram, + allFiles = checksumOpt.flatMap(_.allFiles)) + + /** Returns the data schema of the table, used for reading stats */ + def tableSchema: StructType = metadata.dataSchema + + def outputTableStatsSchema: StructType = metadata.dataSchema + + def outputAttributeSchema: StructType = metadata.dataSchema + + /** Returns the schema of the columns written out to file (overridden in write path) */ + def dataSchema: StructType = metadata.dataSchema + + /** Return the set of properties of the table. */ + def getProperties: mutable.Map[String, String] = { + val base = new mutable.LinkedHashMap[String, String]() + metadata.configuration.foreach { case (k, v) => + if (k != "path") { + base.put(k, v) + } + } + base.put(Protocol.MIN_READER_VERSION_PROP, protocol.minReaderVersion.toString) + base.put(Protocol.MIN_WRITER_VERSION_PROP, protocol.minWriterVersion.toString) + if (protocol.supportsReaderFeatures || protocol.supportsWriterFeatures) { + val features = protocol.readerAndWriterFeatureNames.map(name => + s"${TableFeatureProtocolUtils.FEATURE_PROP_PREFIX}$name" -> + TableFeatureProtocolUtils.FEATURE_PROP_SUPPORTED) + base ++ features.toSeq.sorted + } else { + base + } + } + + /** The [[CheckpointProvider]] for the underlying checkpoint */ + lazy val checkpointProvider: CheckpointProvider = logSegment.checkpointProvider match { + case cp: CheckpointProvider => cp + case uninitializedProvider: UninitializedCheckpointProvider => + CheckpointProvider(spark, this, checksumOpt, uninitializedProvider) + case o => throw new IllegalStateException(s"Unknown checkpoint provider: ${o.getClass.getName}") + } + + def redactedPath: String = + Utils.redact(spark.sessionState.conf.stringRedactionPattern, path.toUri.toString) + + /** + * Ensures that commit files are backfilled up to the current version in the snapshot. + * + * This method checks if there are any un-backfilled versions up to the current version and + * triggers the backfilling process using the commit-owner. It verifies that the delta file for + * the current version exists after the backfilling process. + * + * @throws IllegalStateException + * if the delta file for the current version is not found after backfilling. + */ + def ensureCommitFilesBackfilled(): Unit = { + val tableCommitOwnerClient = tableCommitOwnerClientOpt.getOrElse { + return + } + val minUnbackfilledVersion = DeltaCommitFileProvider(this).minUnbackfilledVersion + if (minUnbackfilledVersion <= version) { + val hadoopConf = deltaLog.newDeltaHadoopConf() + tableCommitOwnerClient.backfillToVersion( + startVersion = minUnbackfilledVersion, endVersion = Some(version)) + val fs = deltaLog.logPath.getFileSystem(hadoopConf) + val expectedBackfilledDeltaFile = FileNames.unsafeDeltaFile(deltaLog.logPath, version) + if (!fs.exists(expectedBackfilledDeltaFile)) { + throw new IllegalStateException("Backfilling of commit files failed. " + + s"Expected delta file $expectedBackfilledDeltaFile not found.") + } + } + } + + + protected def emptyDF: DataFrame = + spark.createDataFrame(spark.sparkContext.emptyRDD[Row], logSchema) + + + override def logInfo(msg: => String): Unit = { + super.logInfo(s"[tableId=${deltaLog.tableId}] " + msg) + } + + override def logWarning(msg: => String): Unit = { + super.logWarning(s"[tableId=${deltaLog.tableId}] " + msg) + } + + override def logWarning(msg: => String, throwable: Throwable): Unit = { + super.logWarning(s"[tableId=${deltaLog.tableId}] " + msg, throwable) + } + + override def logError(msg: => String): Unit = { + super.logError(s"[tableId=${deltaLog.tableId}] " + msg) + } + + override def logError(msg: => String, throwable: Throwable): Unit = { + super.logError(s"[tableId=${deltaLog.tableId}] " + msg, throwable) + } + + override def toString: String = + s"${getClass.getSimpleName}(path=$path, version=$version, metadata=$metadata, " + + s"logSegment=$logSegment, checksumOpt=$checksumOpt)" + + // --- modified start + override def filesForScan(filters: Seq[Expression], keepNumRecords: Boolean): DeltaScan = { + val deltaScan = ClickhouseSnapshot.deltaScanCache.get( + FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), filters, None), + () => { + super.filesForScan(filters, keepNumRecords) + }) + + replaceWithAddMergeTreeParts(deltaScan) + } + + override def filesForScan(limit: Long, partitionFilters: Seq[Expression]): DeltaScan = { + val deltaScan = ClickhouseSnapshot.deltaScanCache.get( + FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), partitionFilters, Some(limit)), + () => { + super.filesForScan(limit, partitionFilters) + }) + + replaceWithAddMergeTreeParts(deltaScan) + } + + private def replaceWithAddMergeTreeParts(deltaScan: DeltaScan) = { + if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { + DeltaScan.apply( + deltaScan.version, + deltaScan.files + .map( + addFile => { + val addFileAsKey = AddFileAsKey(addFile) + + val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) + // this is for later use + ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) + ret + }), + deltaScan.total, + deltaScan.partition, + deltaScan.scanned + )( + deltaScan.scannedSnapshot, + deltaScan.partitionFilters, + deltaScan.dataFilters, + deltaScan.unusedFilters, + deltaScan.scanDurationMs, + deltaScan.dataSkippingType + ) + } else { + deltaScan + } + } + // --- modified end + + logInfo(s"Created snapshot $this") + init() +} + +object Snapshot extends DeltaLogging { + + // Used by [[loadActions]] and [[stateReconstruction]] + val ADD_STATS_TO_USE_COL_NAME = "add_stats_to_use" + + private val defaultNumSnapshotPartitions: Int = 50 + + /** Verifies that a set of delta or checkpoint files to be read actually belongs to this table. */ + private def assertLogFilesBelongToTable(logBasePath: Path, files: Seq[FileStatus]): Unit = { + val logPath = new Path(logBasePath.toUri) + val commitDirPath = FileNames.commitDirPath(logPath) + files.map(_.getPath).foreach { filePath => + val commitParent = new Path(filePath.toUri).getParent + if (commitParent != logPath && commitParent != commitDirPath) { + // scalastyle:off throwerror + throw new AssertionError(s"File ($filePath) doesn't belong in the " + + s"transaction log at $logBasePath.") + // scalastyle:on throwerror + } + } + } +} + +/** + * An initial snapshot with only metadata specified. Useful for creating a DataFrame from an + * existing parquet table during its conversion to delta. + * + * @param logPath the path to transaction log + * @param deltaLog the delta log object + * @param metadata the metadata of the table + */ +class InitialSnapshot( + val logPath: Path, + override val deltaLog: DeltaLog, + override val metadata: Metadata) + extends Snapshot( + path = logPath, + version = -1, + logSegment = LogSegment.empty(logPath), + deltaLog = deltaLog, + checksumOpt = None + ) { + + def this(logPath: Path, deltaLog: DeltaLog) = this( + logPath, + deltaLog, + Metadata( + configuration = DeltaConfigs.mergeGlobalConfigs( + sqlConfs = SparkSession.active.sessionState.conf, + tableConf = Map.empty, + ignoreProtocolConfsOpt = Some( + DeltaConfigs.ignoreProtocolDefaultsIsSet( + sqlConfs = SparkSession.active.sessionState.conf, + tableConf = deltaLog.allOptions))), + createdTime = Some(System.currentTimeMillis()))) + + override def stateDS: Dataset[SingleAction] = emptyDF.as[SingleAction] + override def stateDF: DataFrame = emptyDF + override protected lazy val computedState: SnapshotState = initialState(metadata) + override def protocol: Protocol = computedState.protocol + override protected lazy val getInCommitTimestampOpt: Option[Long] = None + + // The [[InitialSnapshot]] is not backed by any external commit-owner. + override def initializeTableCommitOwner(): Option[TableCommitOwnerClient] = None + override def timestamp: Long = -1L +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala new file mode 100644 index 000000000000..8b4a13a30a69 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta.catalog + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.{ResolvedTable, UnresolvedTable} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.connector.catalog.V1Table +import org.apache.spark.sql.connector.read.InputPartition +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} +import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, DeltaTableUtils, DeltaTimeTravelSpec, Snapshot, UnresolvedPathBasedDeltaTable} +import org.apache.spark.sql.delta.actions.{Metadata, Protocol} +import org.apache.spark.sql.delta.sources.DeltaDataSource +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} +import org.apache.spark.sql.execution.datasources.utils.MergeTreePartsPartitionsUtil +import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat +import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.util.collection.BitSet + +import org.apache.hadoop.fs.Path + +import java.{util => ju} + +import scala.collection.JavaConverters._ + +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) +class ClickHouseTableV2( + override val spark: SparkSession, + override val path: Path, + override val catalogTable: Option[CatalogTable] = None, + override val tableIdentifier: Option[String] = None, + override val timeTravelOpt: Option[DeltaTimeTravelSpec] = None, + override val options: Map[String, String] = Map.empty, + val clickhouseExtensionOptions: Map[String, String] = Map.empty) + extends DeltaTableV2(spark, path, catalogTable, tableIdentifier, timeTravelOpt, options) + with ClickHouseTableV2Base { + + lazy val (rootPath, partitionFilters, timeTravelByPath) = { + if (catalogTable.isDefined) { + // Fast path for reducing path munging overhead + (new Path(catalogTable.get.location), Nil, None) + } else { + DeltaDataSource.parsePathIdentifier(spark, path.toString, options) + } + } + + override def name(): String = + catalogTable + .map(_.identifier.unquotedString) + .orElse(tableIdentifier) + .getOrElse(s"clickhouse.`${deltaLog.dataPath}`") + + private lazy val timeTravelSpec: Option[DeltaTimeTravelSpec] = { + if (timeTravelOpt.isDefined && timeTravelByPath.isDefined) { + throw DeltaErrors.multipleTimeTravelSyntaxUsed + } + timeTravelOpt.orElse(timeTravelByPath) + } + + private lazy val caseInsensitiveOptions = new CaseInsensitiveStringMap(options.asJava) + + override def properties(): ju.Map[String, String] = { + val ret = super.properties() + + // for file path based write + if (initialSnapshot.version < 0 && clickhouseExtensionOptions.nonEmpty) { + ret.putAll(clickhouseExtensionOptions.asJava) + } + ret + } + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + new WriteIntoDeltaBuilder( + this, + info.options, + spark.sessionState.conf.useNullsForMissingDefaultColumnValues) + } + + def getFileFormat(protocol: Protocol, meta: Metadata): DeltaMergeTreeFileFormat = { + new DeltaMergeTreeFileFormat( + protocol, + meta, + dataBaseName, + tableName, + ClickhouseSnapshot.genSnapshotId(initialSnapshot), + orderByKeyOption, + lowCardKeyOption, + minmaxIndexKeyOption, + bfIndexKeyOption, + setIndexKeyOption, + primaryKeyOption, + clickhouseTableConfigs, + partitionColumns + ) + } + + override def deltaProperties(): ju.Map[String, String] = properties() + + override def deltaCatalog(): Option[CatalogTable] = catalogTable + + override def deltaPath(): Path = path + + override def deltaSnapshot(): Snapshot = initialSnapshot + + def cacheThis(): Unit = { + ClickHouseTableV2.deltaLog2Table.put(deltaLog, this) + } + + cacheThis() +} + +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) +class TempClickHouseTableV2( + override val spark: SparkSession, + override val catalogTable: Option[CatalogTable] = None) + extends ClickHouseTableV2(spark, null, catalogTable) { + import collection.JavaConverters._ + override def properties(): ju.Map[String, String] = catalogTable.get.properties.asJava + override lazy val partitionColumns: Seq[String] = catalogTable.get.partitionColumnNames + override def cacheThis(): Unit = {} +} + +object ClickHouseTableV2 extends Logging { + private val deltaLog2Table = + new scala.collection.concurrent.TrieMap[DeltaLog, ClickHouseTableV2]() + // for CTAS use + val temporalThreadLocalCHTable = new ThreadLocal[ClickHouseTableV2]() + + def getTable(deltaLog: DeltaLog): ClickHouseTableV2 = { + if (deltaLog2Table.contains(deltaLog)) { + deltaLog2Table(deltaLog) + } else if (temporalThreadLocalCHTable.get() != null) { + temporalThreadLocalCHTable.get() + } else { + throw new IllegalStateException( + s"Can not find ClickHouseTableV2 for deltalog ${deltaLog.dataPath}") + } + } + + def clearCache(): Unit = { + deltaLog2Table.clear() + temporalThreadLocalCHTable.remove() + } + + def partsPartitions( + deltaLog: DeltaLog, + relation: HadoopFsRelation, + selectedPartitions: Array[PartitionDirectory], + output: Seq[Attribute], + bucketedScan: Boolean, + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + disableBucketedScan: Boolean, + filterExprs: Seq[Expression]): Seq[InputPartition] = { + val tableV2 = ClickHouseTableV2.getTable(deltaLog) + + MergeTreePartsPartitionsUtil.getMergeTreePartsPartitions( + relation, + selectedPartitions, + output, + bucketedScan, + tableV2.spark, + tableV2, + optionalBucketSet, + optionalNumCoalescedBuckets, + disableBucketedScan, + filterExprs) + } + + /** Resolves a path into a DeltaTableV2, leveraging standard v2 table resolution. */ + def apply( + spark: SparkSession, + tablePath: Path, + options: Map[String, String], + cmd: String): DeltaTableV2 = + resolve(spark, UnresolvedPathBasedDeltaTable(tablePath.toString, options, cmd), cmd) + + /** Resolves a table identifier into a DeltaTableV2, leveraging standard v2 table resolution. */ + def apply(spark: SparkSession, tableId: TableIdentifier, cmd: String): DeltaTableV2 = { + resolve(spark, UnresolvedTable(tableId.nameParts, cmd, None), cmd) + } + + /** Applies standard v2 table resolution to an unresolved Delta table plan node */ + def resolve(spark: SparkSession, unresolved: LogicalPlan, cmd: String): DeltaTableV2 = + extractFrom(spark.sessionState.analyzer.ResolveRelations(unresolved), cmd) + + /** + * Extracts the DeltaTableV2 from a resolved Delta table plan node, throwing "table not found" if + * the node does not actually represent a resolved Delta table. + */ + def extractFrom(plan: LogicalPlan, cmd: String): DeltaTableV2 = plan match { + case ResolvedTable(_, _, d: ClickHouseTableV2, _) => d + case ResolvedTable(_, _, d: DeltaTableV2, _) => d + case ResolvedTable(_, _, t: V1Table, _) + if CHDataSourceUtils.isClickHouseTable(t.catalogTable) => + new ClickHouseTableV2(SparkSession.active, new Path(t.v1Table.location), Some(t.v1Table)) + case ResolvedTable(_, _, t: V1Table, _) if DeltaTableUtils.isDeltaTable(t.catalogTable) => + DeltaTableV2(SparkSession.active, new Path(t.v1Table.location), Some(t.v1Table)) + case _ => throw DeltaErrors.notADeltaTableException(cmd) + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala new file mode 100644 index 000000000000..dec1f4b9c3f5 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala @@ -0,0 +1,557 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands + +import java.util.concurrent.TimeUnit + +import org.apache.spark.sql.delta.metric.IncrementMetric +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, AddFile, FileAction} +import org.apache.spark.sql.delta.commands.DeleteCommand.{rewritingFilesMsg, FINDING_TOUCHED_FILES_MSG} +import org.apache.spark.sql.delta.commands.MergeIntoCommandBase.totalBytesAndDistinctPartitionValues +import org.apache.spark.sql.delta.files.TahoeBatchFileIndex +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + +import org.apache.spark.SparkContext +import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, EqualNullSafe, Expression, If, Literal, Not} +import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral +import org.apache.spark.sql.catalyst.plans.QueryPlan +import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} +import org.apache.spark.sql.execution.command.LeafRunnableCommand +import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} +import org.apache.spark.sql.functions.{col, explode, input_file_name, split} +import org.apache.spark.sql.types.LongType + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0. It is modified to overcome the following issues: + * 1. In Clickhouse backend, we can't implement input_file_name() correctly, we can only implement + * it so that it return a a list of filenames (concated by ','). + */ + +trait DeleteCommandMetrics { self: LeafRunnableCommand => + @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() + + def createMetrics: Map[String, SQLMetric] = Map[String, SQLMetric]( + "numRemovedFiles" -> createMetric(sc, "number of files removed."), + "numAddedFiles" -> createMetric(sc, "number of files added."), + "numDeletedRows" -> createMetric(sc, "number of rows deleted."), + "numFilesBeforeSkipping" -> createMetric(sc, "number of files before skipping"), + "numBytesBeforeSkipping" -> createMetric(sc, "number of bytes before skipping"), + "numFilesAfterSkipping" -> createMetric(sc, "number of files after skipping"), + "numBytesAfterSkipping" -> createMetric(sc, "number of bytes after skipping"), + "numPartitionsAfterSkipping" -> createMetric(sc, "number of partitions after skipping"), + "numPartitionsAddedTo" -> createMetric(sc, "number of partitions added"), + "numPartitionsRemovedFrom" -> createMetric(sc, "number of partitions removed"), + "numCopiedRows" -> createMetric(sc, "number of rows copied"), + "numAddedBytes" -> createMetric(sc, "number of bytes added"), + "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), + "executionTimeMs" -> + createTimingMetric(sc, "time taken to execute the entire operation"), + "scanTimeMs" -> + createTimingMetric(sc, "time taken to scan the files for matches"), + "rewriteTimeMs" -> + createTimingMetric(sc, "time taken to rewrite the matched files"), + "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), + "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), + "numTouchedRows" -> createMetric(sc, "number of rows touched"), + "numDeletionVectorsAdded" -> createMetric(sc, "number of deletion vectors added"), + "numDeletionVectorsRemoved" -> createMetric(sc, "number of deletion vectors removed"), + "numDeletionVectorsUpdated" -> createMetric(sc, "number of deletion vectors updated") + ) + + def getDeletedRowsFromAddFilesAndUpdateMetrics(files: Seq[AddFile]) : Option[Long] = { + if (!conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA)) { + return None; + } + // No file to get metadata, return none to be consistent with metadata stats disabled + if (files.isEmpty) { + return None + } + // Return None if any file does not contain numLogicalRecords status + var count: Long = 0 + for (file <- files) { + if (file.numLogicalRecords.isEmpty) { + return None + } + count += file.numLogicalRecords.get + } + metrics("numDeletedRows").set(count) + return Some(count) + } +} + +/** + * Performs a Delete based on the search condition + * + * Algorithm: + * 1) Scan all the files and determine which files have + * the rows that need to be deleted. + * 2) Traverse the affected files and rebuild the touched files. + * 3) Use the Delta protocol to atomically write the remaining rows to new files and remove + * the affected files that are identified in step 1. + */ +case class DeleteCommand( + deltaLog: DeltaLog, + catalogTable: Option[CatalogTable], + target: LogicalPlan, + condition: Option[Expression]) + extends LeafRunnableCommand with DeltaCommand with DeleteCommandMetrics { + + override def innerChildren: Seq[QueryPlan[_]] = Seq(target) + + override val output: Seq[Attribute] = Seq(AttributeReference("num_affected_rows", LongType)()) + + override lazy val metrics = createMetrics + + final override def run(sparkSession: SparkSession): Seq[Row] = { + recordDeltaOperation(deltaLog, "delta.dml.delete") { + deltaLog.withNewTransaction(catalogTable) { txn => + DeltaLog.assertRemovable(txn.snapshot) + if (hasBeenExecuted(txn, sparkSession)) { + sendDriverMetrics(sparkSession, metrics) + return Seq.empty + } + + val deleteActions = performDelete(sparkSession, deltaLog, txn) + txn.commitIfNeeded(actions = deleteActions, + op = DeltaOperations.Delete(condition.toSeq), + tags = RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot)) + } + // Re-cache all cached plans(including this relation itself, if it's cached) that refer to + // this data source relation. + sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) + } + + // Adjust for deletes at partition boundaries. Deletes at partition boundaries is a metadata + // operation, therefore we don't actually have any information around how many rows were deleted + // While this info may exist in the file statistics, it's not guaranteed that we have these + // statistics. To avoid any performance regressions, we currently just return a -1 in such cases + if (metrics("numRemovedFiles").value > 0 && metrics("numDeletedRows").value == 0) { + Seq(Row(-1L)) + } else { + Seq(Row(metrics("numDeletedRows").value)) + } + } + + def performDelete( + sparkSession: SparkSession, + deltaLog: DeltaLog, + txn: OptimisticTransaction): Seq[Action] = { + import org.apache.spark.sql.delta.implicits._ + + var numRemovedFiles: Long = 0 + var numAddedFiles: Long = 0 + var numAddedChangeFiles: Long = 0 + var scanTimeMs: Long = 0 + var rewriteTimeMs: Long = 0 + var numAddedBytes: Long = 0 + var changeFileBytes: Long = 0 + var numRemovedBytes: Long = 0 + var numFilesBeforeSkipping: Long = 0 + var numBytesBeforeSkipping: Long = 0 + var numFilesAfterSkipping: Long = 0 + var numBytesAfterSkipping: Long = 0 + var numPartitionsAfterSkipping: Option[Long] = None + var numPartitionsRemovedFrom: Option[Long] = None + var numPartitionsAddedTo: Option[Long] = None + var numDeletedRows: Option[Long] = None + var numCopiedRows: Option[Long] = None + var numDeletionVectorsAdded: Long = 0 + var numDeletionVectorsRemoved: Long = 0 + var numDeletionVectorsUpdated: Long = 0 + + val startTime = System.nanoTime() + val numFilesTotal = txn.snapshot.numOfFiles + + val deleteActions: Seq[Action] = condition match { + case None => + // Case 1: Delete the whole table if the condition is true + val reportRowLevelMetrics = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) + val allFiles = txn.filterFiles(Nil, keepNumRecords = reportRowLevelMetrics) + + numRemovedFiles = allFiles.size + numDeletionVectorsRemoved = allFiles.count(_.deletionVector != null) + scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 + val (numBytes, numPartitions) = totalBytesAndDistinctPartitionValues(allFiles) + numRemovedBytes = numBytes + numFilesBeforeSkipping = numRemovedFiles + numBytesBeforeSkipping = numBytes + numFilesAfterSkipping = numRemovedFiles + numBytesAfterSkipping = numBytes + numDeletedRows = getDeletedRowsFromAddFilesAndUpdateMetrics(allFiles) + + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsAfterSkipping = Some(numPartitions) + numPartitionsRemovedFrom = Some(numPartitions) + numPartitionsAddedTo = Some(0) + } + val operationTimestamp = System.currentTimeMillis() + allFiles.map(_.removeWithTimestamp(operationTimestamp)) + case Some(cond) => + val (metadataPredicates, otherPredicates) = + DeltaTableUtils.splitMetadataAndDataPredicates( + cond, txn.metadata.partitionColumns, sparkSession) + + numFilesBeforeSkipping = txn.snapshot.numOfFiles + numBytesBeforeSkipping = txn.snapshot.sizeInBytes + + if (otherPredicates.isEmpty) { + // Case 2: The condition can be evaluated using metadata only. + // Delete a set of files without the need of scanning any data files. + val operationTimestamp = System.currentTimeMillis() + val reportRowLevelMetrics = conf.getConf(DeltaSQLConf.DELTA_DML_METRICS_FROM_METADATA) + val candidateFiles = + txn.filterFiles(metadataPredicates, keepNumRecords = reportRowLevelMetrics) + + scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 + numRemovedFiles = candidateFiles.size + numRemovedBytes = candidateFiles.map(_.size).sum + numFilesAfterSkipping = candidateFiles.size + numDeletionVectorsRemoved = candidateFiles.count(_.deletionVector != null) + val (numCandidateBytes, numCandidatePartitions) = + totalBytesAndDistinctPartitionValues(candidateFiles) + numBytesAfterSkipping = numCandidateBytes + numDeletedRows = getDeletedRowsFromAddFilesAndUpdateMetrics(candidateFiles) + + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsAfterSkipping = Some(numCandidatePartitions) + numPartitionsRemovedFrom = Some(numCandidatePartitions) + numPartitionsAddedTo = Some(0) + } + candidateFiles.map(_.removeWithTimestamp(operationTimestamp)) + } else { + // Case 3: Delete the rows based on the condition. + + // Should we write the DVs to represent the deleted rows? + val shouldWriteDVs = shouldWritePersistentDeletionVectors(sparkSession, txn) + + val candidateFiles = txn.filterFiles( + metadataPredicates ++ otherPredicates, + keepNumRecords = shouldWriteDVs) + // `candidateFiles` contains the files filtered using statistics and delete condition + // They may or may not contains any rows that need to be deleted. + + numFilesAfterSkipping = candidateFiles.size + val (numCandidateBytes, numCandidatePartitions) = + totalBytesAndDistinctPartitionValues(candidateFiles) + numBytesAfterSkipping = numCandidateBytes + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsAfterSkipping = Some(numCandidatePartitions) + } + + val nameToAddFileMap = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) + + val fileIndex = new TahoeBatchFileIndex( + sparkSession, "delete", candidateFiles, deltaLog, deltaLog.dataPath, txn.snapshot) + if (shouldWriteDVs) { + val targetDf = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( + sparkSession, + target, + fileIndex) + + // Does the target table already has DVs enabled? If so, we need to read the table + // with deletion vectors. + val mustReadDeletionVectors = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) + + val touchedFiles = DMLWithDeletionVectorsHelper.findTouchedFiles( + sparkSession, + txn, + mustReadDeletionVectors, + deltaLog, + targetDf, + fileIndex, + cond, + opName = "DELETE") + + if (touchedFiles.nonEmpty) { + val (actions, metricMap) = DMLWithDeletionVectorsHelper.processUnmodifiedData( + sparkSession, + touchedFiles, + txn.snapshot) + metrics("numDeletedRows").set(metricMap("numModifiedRows")) + numDeletionVectorsAdded = metricMap("numDeletionVectorsAdded") + numDeletionVectorsRemoved = metricMap("numDeletionVectorsRemoved") + numDeletionVectorsUpdated = metricMap("numDeletionVectorsUpdated") + numRemovedFiles = metricMap("numRemovedFiles") + actions + } else { + Nil // Nothing to update + } + } else { + // Keep everything from the resolved target except a new TahoeFileIndex + // that only involves the affected files instead of all files. + val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) + val data = Dataset.ofRows(sparkSession, newTarget) + val incrDeletedCountExpr = IncrementMetric(TrueLiteral, metrics("numDeletedRows")) + val filesToRewrite = + withStatusCode("DELTA", FINDING_TOUCHED_FILES_MSG) { + if (candidateFiles.isEmpty) { + Array.empty[String] + } else { + // --- modified start + data.filter(new Column(cond)) + .select(input_file_name().as("input_files")) + .filter(new Column(incrDeletedCountExpr)) + .select(explode(split(col("input_files"), ","))) + .distinct() + .as[String] + .collect() + // --- modified end + } + } + + numRemovedFiles = filesToRewrite.length + scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 + if (filesToRewrite.isEmpty) { + // Case 3.1: no row matches and no delete will be triggered + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsRemovedFrom = Some(0) + numPartitionsAddedTo = Some(0) + } + Nil + } else { + // Case 3.2: some files need an update to remove the deleted files + // Do the second pass and just read the affected files + val baseRelation = buildBaseRelation( + sparkSession, txn, "delete", deltaLog.dataPath, filesToRewrite, nameToAddFileMap) + // Keep everything from the resolved target except a new TahoeFileIndex + // that only involves the affected files instead of all files. + val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) + val targetDF = RowTracking.preserveRowTrackingColumns( + dfWithoutRowTrackingColumns = Dataset.ofRows(sparkSession, newTarget), + snapshot = txn.snapshot) + val filterCond = Not(EqualNullSafe(cond, Literal.TrueLiteral)) + val rewrittenActions = rewriteFiles(txn, targetDF, filterCond, filesToRewrite.length) + val (changeFiles, rewrittenFiles) = rewrittenActions + .partition(_.isInstanceOf[AddCDCFile]) + numAddedFiles = rewrittenFiles.size + val removedFiles = filesToRewrite.map(f => + getTouchedFile(deltaLog.dataPath, f, nameToAddFileMap)) + val (removedBytes, removedPartitions) = + totalBytesAndDistinctPartitionValues(removedFiles) + numRemovedBytes = removedBytes + val (rewrittenBytes, rewrittenPartitions) = + totalBytesAndDistinctPartitionValues(rewrittenFiles) + numAddedBytes = rewrittenBytes + if (txn.metadata.partitionColumns.nonEmpty) { + numPartitionsRemovedFrom = Some(removedPartitions) + numPartitionsAddedTo = Some(rewrittenPartitions) + } + numAddedChangeFiles = changeFiles.size + changeFileBytes = changeFiles.collect { case f: AddCDCFile => f.size }.sum + rewriteTimeMs = + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) - scanTimeMs + numDeletedRows = Some(metrics("numDeletedRows").value) + numCopiedRows = + Some(metrics("numTouchedRows").value - metrics("numDeletedRows").value) + numDeletionVectorsRemoved = removedFiles.count(_.deletionVector != null) + val operationTimestamp = System.currentTimeMillis() + removeFilesFromPaths( + deltaLog, nameToAddFileMap, filesToRewrite, operationTimestamp) ++ rewrittenActions + } + } + } + } + metrics("numRemovedFiles").set(numRemovedFiles) + metrics("numAddedFiles").set(numAddedFiles) + val executionTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 + metrics("executionTimeMs").set(executionTimeMs) + metrics("scanTimeMs").set(scanTimeMs) + metrics("rewriteTimeMs").set(rewriteTimeMs) + metrics("numAddedChangeFiles").set(numAddedChangeFiles) + metrics("changeFileBytes").set(changeFileBytes) + metrics("numAddedBytes").set(numAddedBytes) + metrics("numRemovedBytes").set(numRemovedBytes) + metrics("numFilesBeforeSkipping").set(numFilesBeforeSkipping) + metrics("numBytesBeforeSkipping").set(numBytesBeforeSkipping) + metrics("numFilesAfterSkipping").set(numFilesAfterSkipping) + metrics("numBytesAfterSkipping").set(numBytesAfterSkipping) + metrics("numDeletionVectorsAdded").set(numDeletionVectorsAdded) + metrics("numDeletionVectorsRemoved").set(numDeletionVectorsRemoved) + metrics("numDeletionVectorsUpdated").set(numDeletionVectorsUpdated) + numPartitionsAfterSkipping.foreach(metrics("numPartitionsAfterSkipping").set) + numPartitionsAddedTo.foreach(metrics("numPartitionsAddedTo").set) + numPartitionsRemovedFrom.foreach(metrics("numPartitionsRemovedFrom").set) + numCopiedRows.foreach(metrics("numCopiedRows").set) + txn.registerSQLMetrics(sparkSession, metrics) + sendDriverMetrics(sparkSession, metrics) + + recordDeltaEvent( + deltaLog, + "delta.dml.delete.stats", + data = DeleteMetric( + condition = condition.map(_.sql).getOrElse("true"), + numFilesTotal, + numFilesAfterSkipping, + numAddedFiles, + numRemovedFiles, + numAddedFiles, + numAddedChangeFiles = numAddedChangeFiles, + numFilesBeforeSkipping, + numBytesBeforeSkipping, + numFilesAfterSkipping, + numBytesAfterSkipping, + numPartitionsAfterSkipping, + numPartitionsAddedTo, + numPartitionsRemovedFrom, + numCopiedRows, + numDeletedRows, + numAddedBytes, + numRemovedBytes, + changeFileBytes = changeFileBytes, + scanTimeMs, + rewriteTimeMs, + numDeletionVectorsAdded, + numDeletionVectorsRemoved, + numDeletionVectorsUpdated) + ) + + if (deleteActions.nonEmpty) { + createSetTransaction(sparkSession, deltaLog).toSeq ++ deleteActions + } else { + Seq.empty + } + } + + /** + * Returns the list of [[AddFile]]s and [[AddCDCFile]]s that have been re-written. + */ + private def rewriteFiles( + txn: OptimisticTransaction, + baseData: DataFrame, + filterCondition: Expression, + numFilesToRewrite: Long): Seq[FileAction] = { + val shouldWriteCdc = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) + + // number of total rows that we have seen / are either copying or deleting (sum of both). + val incrTouchedCountExpr = IncrementMetric(TrueLiteral, metrics("numTouchedRows")) + + withStatusCode( + "DELTA", rewritingFilesMsg(numFilesToRewrite)) { + val dfToWrite = if (shouldWriteCdc) { + import org.apache.spark.sql.delta.commands.cdc.CDCReader._ + // The logic here ends up being surprisingly elegant, with all source rows ending up in + // the output. Recall that we flipped the user-provided delete condition earlier, before the + // call to `rewriteFiles`. All rows which match this latest `filterCondition` are retained + // as table data, while all rows which don't match are removed from the rewritten table data + // but do get included in the output as CDC events. + baseData + .filter(new Column(incrTouchedCountExpr)) + .withColumn( + CDC_TYPE_COLUMN_NAME, + new Column(If(filterCondition, CDC_TYPE_NOT_CDC, CDC_TYPE_DELETE)) + ) + } else { + baseData + .filter(new Column(incrTouchedCountExpr)) + .filter(new Column(filterCondition)) + } + + txn.writeFiles(dfToWrite) + } + } + + def shouldWritePersistentDeletionVectors( + spark: SparkSession, txn: OptimisticTransaction): Boolean = { + spark.conf.get(DeltaSQLConf.DELETE_USE_PERSISTENT_DELETION_VECTORS) && + DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) + } +} + +object DeleteCommand { + def apply(delete: DeltaDelete): DeleteCommand = { + EliminateSubqueryAliases(delete.child) match { + case DeltaFullTable(relation, fileIndex) => + DeleteCommand(fileIndex.deltaLog, relation.catalogTable, delete.child, delete.condition) + case o => + throw DeltaErrors.notADeltaSourceException("DELETE", Some(o)) + } + } + + val FILE_NAME_COLUMN: String = "_input_file_name_" + val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for DELETE operation" + + def rewritingFilesMsg(numFilesToRewrite: Long): String = + s"Rewriting $numFilesToRewrite files for DELETE operation" +} + +/** + * Used to report details about delete. + * + * @param condition: what was the delete condition + * @param numFilesTotal: how big is the table + * @param numTouchedFiles: how many files did we touch. Alias for `numFilesAfterSkipping` + * @param numRewrittenFiles: how many files had to be rewritten. Alias for `numAddedFiles` + * @param numRemovedFiles: how many files we removed. Alias for `numTouchedFiles` + * @param numAddedFiles: how many files we added. Alias for `numRewrittenFiles` + * @param numAddedChangeFiles: how many change files were generated + * @param numFilesBeforeSkipping: how many candidate files before skipping + * @param numBytesBeforeSkipping: how many candidate bytes before skipping + * @param numFilesAfterSkipping: how many candidate files after skipping + * @param numBytesAfterSkipping: how many candidate bytes after skipping + * @param numPartitionsAfterSkipping: how many candidate partitions after skipping + * @param numPartitionsAddedTo: how many new partitions were added + * @param numPartitionsRemovedFrom: how many partitions were removed + * @param numCopiedRows: how many rows were copied + * @param numDeletedRows: how many rows were deleted + * @param numBytesAdded: how many bytes were added + * @param numBytesRemoved: how many bytes were removed + * @param changeFileBytes: total size of change files generated + * @param scanTimeMs: how long did finding take + * @param rewriteTimeMs: how long did rewriting take + * @param numDeletionVectorsAdded: how many deletion vectors were added + * @param numDeletionVectorsRemoved: how many deletion vectors were removed + * @param numDeletionVectorsUpdated: how many deletion vectors were updated + * + * @note All the time units are milliseconds. + */ +case class DeleteMetric( + condition: String, + numFilesTotal: Long, + numTouchedFiles: Long, + numRewrittenFiles: Long, + numRemovedFiles: Long, + numAddedFiles: Long, + numAddedChangeFiles: Long, + numFilesBeforeSkipping: Long, + numBytesBeforeSkipping: Long, + numFilesAfterSkipping: Long, + numBytesAfterSkipping: Long, + numPartitionsAfterSkipping: Option[Long], + numPartitionsAddedTo: Option[Long], + numPartitionsRemovedFrom: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + numCopiedRows: Option[Long], + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + numDeletedRows: Option[Long], + numBytesAdded: Long, + numBytesRemoved: Long, + changeFileBytes: Long, + scanTimeMs: Long, + rewriteTimeMs: Long, + numDeletionVectorsAdded: Long, + numDeletionVectorsRemoved: Long, + numDeletionVectorsUpdated: Long +) diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala new file mode 100644 index 000000000000..5b2170220228 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -0,0 +1,608 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands + +import java.util.ConcurrentModificationException + +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.sql.delta.skipping.MultiDimClustering +import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumnInfo} +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.DeltaOperations.Operation +import org.apache.spark.sql.delta.actions.{Action, AddFile, DeletionVectorDescriptor, FileAction, RemoveFile} +import org.apache.spark.sql.delta.commands.optimize._ +import org.apache.spark.sql.delta.files.SQLMetricsReporting +import org.apache.spark.sql.delta.schema.SchemaUtils +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +import org.apache.spark.SparkContext +import org.apache.spark.SparkContext.SPARK_JOB_GROUP_ID +import org.apache.spark.sql.{AnalysisException, Encoders, Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} +import org.apache.spark.sql.execution.command.RunnableCommand +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric +import org.apache.spark.sql.types._ +import org.apache.spark.util.{SystemClock, ThreadUtils} + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0. It is modified in: + * 1. getDeltaTable supports to get ClickHouseTableV2 + * 2. runOptimizeBinJobClickhouse + * 3. groupFilesIntoBinsClickhouse + */ + +/** Base class defining abstract optimize command */ +abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaCommand { + + override val output: Seq[Attribute] = Seq( + AttributeReference("path", StringType)(), + AttributeReference("metrics", Encoders.product[OptimizeMetrics].schema)()) + + /** + * Validates ZOrderBy columns + * - validates that partitions columns are not used in `unresolvedZOrderByCols` + * - validates that we already collect stats for all the columns used in `unresolvedZOrderByCols` + * + * @param spark [[SparkSession]] to use + * @param txn the [[OptimisticTransaction]] being used to optimize + * @param unresolvedZOrderByCols Seq of [[UnresolvedAttribute]] corresponding to zOrderBy columns + */ + def validateZorderByColumns( + spark: SparkSession, + txn: OptimisticTransaction, + unresolvedZOrderByCols: Seq[UnresolvedAttribute]): Unit = { + if (unresolvedZOrderByCols.isEmpty) return + val metadata = txn.snapshot.metadata + val partitionColumns = metadata.partitionColumns.toSet + val dataSchema = + StructType(metadata.schema.filterNot(c => partitionColumns.contains(c.name))) + val df = spark.createDataFrame(new java.util.ArrayList[Row](), dataSchema) + val checkColStat = spark.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK) + val statCollectionSchema = txn.snapshot.statCollectionLogicalSchema + val colsWithoutStats = ArrayBuffer[String]() + + unresolvedZOrderByCols.foreach { colAttribute => + val colName = colAttribute.name + if (checkColStat) { + try { + SchemaUtils.findColumnPosition(colAttribute.nameParts, statCollectionSchema) + } catch { + case e: AnalysisException if e.getMessage.contains("Couldn't find column") => + colsWithoutStats.append(colName) + } + } + val isNameEqual = spark.sessionState.conf.resolver + if (partitionColumns.find(isNameEqual(_, colName)).nonEmpty) { + throw DeltaErrors.zOrderingOnPartitionColumnException(colName) + } + if (df.queryExecution.analyzed.resolve(colAttribute.nameParts, isNameEqual).isEmpty) { + throw DeltaErrors.zOrderingColumnDoesNotExistException(colName) + } + } + if (checkColStat && colsWithoutStats.nonEmpty) { + throw DeltaErrors.zOrderingOnColumnWithNoStatsException( + colsWithoutStats.toSeq, spark) + } + } +} + +object OptimizeTableCommand { + /** + * Alternate constructor that converts a provided path or table identifier into the + * correct child LogicalPlan node. If both path and tableIdentifier are specified (or + * if both are None), this method will throw an exception. If a table identifier is + * specified, the child LogicalPlan will be an [[UnresolvedTable]] whereas if a path + * is specified, it will be an [[UnresolvedPathBasedDeltaTable]]. + * + * Note that the returned OptimizeTableCommand will have an *unresolved* child table + * and hence, the command needs to be analyzed before it can be executed. + */ + def apply( + path: Option[String], + tableIdentifier: Option[TableIdentifier], + userPartitionPredicates: Seq[String], + optimizeContext: DeltaOptimizeContext = DeltaOptimizeContext())( + zOrderBy: Seq[UnresolvedAttribute]): OptimizeTableCommand = { + val plan = UnresolvedDeltaPathOrIdentifier(path, tableIdentifier, "OPTIMIZE") + OptimizeTableCommand(plan, userPartitionPredicates, optimizeContext)(zOrderBy) + } +} + +/** + * The `optimize` command implementation for Spark SQL. Example SQL: + * {{{ + * OPTIMIZE ('/path/to/dir' | delta.table) [WHERE part = 25]; + * }}} + */ +case class OptimizeTableCommand( + override val child: LogicalPlan, + userPartitionPredicates: Seq[String], + optimizeContext: DeltaOptimizeContext +)(val zOrderBy: Seq[UnresolvedAttribute]) + extends OptimizeTableCommandBase with RunnableCommand with UnaryNode { + + override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil + + override protected def withNewChildInternal(newChild: LogicalPlan): OptimizeTableCommand = + copy(child = newChild)(zOrderBy) + + override def run(sparkSession: SparkSession): Seq[Row] = { + // --- modified start + val table = OptimizeTableCommandOverwrites.getDeltaTable(child, "OPTIMIZE") + // --- modified end + + val txn = table.startTransaction() + if (txn.readVersion == -1) { + throw DeltaErrors.notADeltaTableException(table.deltaLog.dataPath.toString) + } + + if (ClusteredTableUtils.isSupported(txn.protocol)) { + if (userPartitionPredicates.nonEmpty) { + throw DeltaErrors.clusteringWithPartitionPredicatesException(userPartitionPredicates) + } + if (zOrderBy.nonEmpty) { + throw DeltaErrors.clusteringWithZOrderByException(zOrderBy) + } + } + + val partitionColumns = txn.snapshot.metadata.partitionColumns + // Parse the predicate expression into Catalyst expression and verify only simple filters + // on partition columns are present + + val partitionPredicates = userPartitionPredicates.flatMap { predicate => + val predicates = parsePredicates(sparkSession, predicate) + verifyPartitionPredicates( + sparkSession, + partitionColumns, + predicates) + predicates + } + + validateZorderByColumns(sparkSession, txn, zOrderBy) + val zOrderByColumns = zOrderBy.map(_.name).toSeq + + new OptimizeExecutor( + sparkSession, + txn, + partitionPredicates, + zOrderByColumns, + isAutoCompact = false, + optimizeContext + ).optimize() + } +} + +/** + * Stored all runtime context information that can control the execution of optimize. + * + * @param reorg The REORG operation that triggered the rewriting task, if any. + * @param minFileSize Files which are smaller than this threshold will be selected for compaction. + * If not specified, [[DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE]] will be used. + * This parameter must be set to `0` when [[reorg]] is set. + * @param maxDeletedRowsRatio Files with a ratio of soft-deleted rows to the total rows larger than + * this threshold will be rewritten by the OPTIMIZE command. If not + * specified, [[DeltaSQLConf.DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO]] + * will be used. This parameter must be set to `0` when [[reorg]] is set. + */ +case class DeltaOptimizeContext( + reorg: Option[DeltaReorgOperation] = None, + minFileSize: Option[Long] = None, + maxFileSize: Option[Long] = None, + maxDeletedRowsRatio: Option[Double] = None) { + if (reorg.nonEmpty) { + require( + minFileSize.contains(0L) && maxDeletedRowsRatio.contains(0d), + "minFileSize and maxDeletedRowsRatio must be 0 when running REORG TABLE.") + } +} + +/** + * Optimize job which compacts small files into larger files to reduce + * the number of files and potentially allow more efficient reads. + * + * @param sparkSession Spark environment reference. + * @param txn The transaction used to optimize this table + * @param partitionPredicate List of partition predicates to select subset of files to optimize. + */ +class OptimizeExecutor( + sparkSession: SparkSession, + txn: OptimisticTransaction, + partitionPredicate: Seq[Expression], + zOrderByColumns: Seq[String], + isAutoCompact: Boolean, + optimizeContext: DeltaOptimizeContext) + extends DeltaCommand with SQLMetricsReporting with Serializable { + + /** + * In which mode the Optimize command is running. There are three valid modes: + * 1. Compaction + * 2. ZOrder + * 3. Clustering + */ + private val optimizeStrategy = + OptimizeTableStrategy(sparkSession, txn.snapshot, optimizeContext, zOrderByColumns) + + /** Timestamp to use in [[FileAction]] */ + private val operationTimestamp = new SystemClock().getTimeMillis() + + private val isClusteredTable = ClusteredTableUtils.isSupported(txn.snapshot.protocol) + + private val isMultiDimClustering = + optimizeStrategy.isInstanceOf[ClusteringStrategy] || + optimizeStrategy.isInstanceOf[ZOrderStrategy] + + private val clusteringColumns: Seq[String] = { + if (zOrderByColumns.nonEmpty) { + zOrderByColumns + } else if (isClusteredTable) { + ClusteringColumnInfo.extractLogicalNames(txn.snapshot) + } else { + Nil + } + } + + def optimize(): Seq[Row] = { + recordDeltaOperation(txn.deltaLog, "delta.optimize") { + + // --- modified start + val isMergeTreeFormat = ClickHouseConfig + .isMergeTreeFormatEngine(txn.deltaLog.unsafeVolatileMetadata.configuration) + // --- modified end + + val minFileSize = optimizeContext.minFileSize.getOrElse( + sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE)) + val maxFileSize = optimizeContext.maxFileSize.getOrElse( + sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE)) + val maxDeletedRowsRatio = optimizeContext.maxDeletedRowsRatio.getOrElse( + sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_DELETED_ROWS_RATIO)) + + val candidateFiles = txn.filterFiles(partitionPredicate, keepNumRecords = true) + val partitionSchema = txn.metadata.partitionSchema + + val filesToProcess = optimizeContext.reorg match { + case Some(reorgOperation) => reorgOperation.filterFilesToReorg(txn.snapshot, candidateFiles) + case None => filterCandidateFileList(minFileSize, maxDeletedRowsRatio, candidateFiles) + } + // --- modified start + val maxThreads = + sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_THREADS) + val (updates, jobs) = if (isMergeTreeFormat) { + val partitionsToCompact = filesToProcess + .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) + .toSeq + val jobs = OptimizeTableCommandOverwrites + .groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) + val updates = ThreadUtils.parmap(jobs, "OptimizeJob", maxThreads) { + partitionBinGroup => + // --- modified start + OptimizeTableCommandOverwrites.runOptimizeBinJobClickhouse( + txn, + partitionBinGroup._1._2, + partitionBinGroup._1._1, + partitionBinGroup._2, + maxFileSize) + // --- modified end + }.flatten + // uniform the jobs type + (updates, jobs.map(v => (v._1._2 ++ Map("bucketNum" -> v._1.toString()), v._2))) + } else { + val partitionsToCompact = filesToProcess.groupBy(_.partitionValues).toSeq + + val jobs = groupFilesIntoBins(partitionsToCompact) + + val updates = ThreadUtils.parmap(jobs, "OptimizeJob", maxThreads) { partitionBinGroup => + runOptimizeBinJob(txn, partitionBinGroup._1, partitionBinGroup._2, maxFileSize) + }.flatten + (updates, jobs) + } + // --- modified end + + val addedFiles = updates.collect { case a: AddFile => a } + val removedFiles = updates.collect { case r: RemoveFile => r } + val removedDVs = filesToProcess.filter(_.deletionVector != null).map(_.deletionVector).toSeq + if (addedFiles.size > 0) { + val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles, removedDVs) + commitAndRetry(txn, getOperation(), updates, metrics) { newTxn => + val newPartitionSchema = newTxn.metadata.partitionSchema + val candidateSetOld = candidateFiles.map(_.path).toSet + val candidateSetNew = newTxn.filterFiles(partitionPredicate).map(_.path).toSet + + // As long as all of the files that we compacted are still part of the table, + // and the partitioning has not changed it is valid to continue to try + // and commit this checkpoint. + if (candidateSetOld.subsetOf(candidateSetNew) && partitionSchema == newPartitionSchema) { + true + } else { + val deleted = candidateSetOld -- candidateSetNew + logWarning(s"The following compacted files were delete " + + s"during checkpoint ${deleted.mkString(",")}. Aborting the compaction.") + false + } + } + } + + val optimizeStats = OptimizeStats() + optimizeStats.addedFilesSizeStats.merge(addedFiles) + optimizeStats.removedFilesSizeStats.merge(removedFiles) + optimizeStats.numPartitionsOptimized = jobs.map(j => j._1).distinct.size + optimizeStats.numBatches = jobs.size + optimizeStats.totalConsideredFiles = candidateFiles.size + optimizeStats.totalFilesSkipped = optimizeStats.totalConsideredFiles - removedFiles.size + optimizeStats.totalClusterParallelism = sparkSession.sparkContext.defaultParallelism + val numTableColumns = txn.snapshot.metadata.schema.size + optimizeStats.numTableColumns = numTableColumns + optimizeStats.numTableColumnsWithStats = + DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(txn.snapshot.metadata) + .min(numTableColumns) + if (removedDVs.size > 0) { + optimizeStats.deletionVectorStats = Some(DeletionVectorStats( + numDeletionVectorsRemoved = removedDVs.size, + numDeletionVectorRowsRemoved = removedDVs.map(_.cardinality).sum)) + } + + optimizeStrategy.updateOptimizeStats(optimizeStats, removedFiles, jobs) + + return Seq(Row(txn.deltaLog.dataPath.toString, optimizeStats.toOptimizeMetrics)) + } + } + + /** + * Helper method to prune the list of selected files based on fileSize and ratio of + * deleted rows according to the deletion vector in [[AddFile]]. + */ + private def filterCandidateFileList( + minFileSize: Long, maxDeletedRowsRatio: Double, files: Seq[AddFile]): Seq[AddFile] = { + + // Select all files in case of multi-dimensional clustering + if (isMultiDimClustering) return files + + def shouldCompactBecauseOfDeletedRows(file: AddFile): Boolean = { + // Always compact files with DVs but without numRecords stats. + // This may be overly aggressive, but it fixes the problem in the long-term, + // as the compacted files will have stats. + (file.deletionVector != null && file.numPhysicalRecords.isEmpty) || + file.deletedToPhysicalRecordsRatio.getOrElse(0d) > maxDeletedRowsRatio + } + + // Select files that are small or have too many deleted rows + files.filter( + addFile => addFile.size < minFileSize || shouldCompactBecauseOfDeletedRows(addFile)) + } + + /** + * Utility methods to group files into bins for optimize. + * + * @param partitionsToCompact List of files to compact group by partition. + * Partition is defined by the partition values (partCol -> partValue) + * @return Sequence of bins. Each bin contains one or more files from the same + * partition and targeted for one output file. + */ + private def groupFilesIntoBins( + partitionsToCompact: Seq[(Map[String, String], Seq[AddFile])]) + : Seq[(Map[String, String], Seq[AddFile])] = { + val maxBinSize = optimizeStrategy.maxBinSize + partitionsToCompact.flatMap { + case (partition, files) => + val bins = new ArrayBuffer[Seq[AddFile]]() + + val currentBin = new ArrayBuffer[AddFile]() + var currentBinSize = 0L + + val preparedFiles = optimizeStrategy.prepareFilesPerPartition(files) + preparedFiles.foreach { file => + // Generally, a bin is a group of existing files, whose total size does not exceed the + // desired maxBinSize. The output file size depends on the mode: + // 1. Compaction: Files in a bin will be coalesced into a single output file. + // 2. ZOrder: all files in a partition will be read by the + // same job, the data will be range-partitioned and + // numFiles = totalFileSize / maxFileSize will be produced. + // 3. Clustering: Files in a bin belongs to one ZCUBE, the data will be + // range-partitioned and numFiles = totalFileSize / maxFileSize. + if (file.size + currentBinSize > maxBinSize) { + bins += currentBin.toVector + currentBin.clear() + currentBin += file + currentBinSize = file.size + } else { + currentBin += file + currentBinSize += file.size + } + } + + if (currentBin.nonEmpty) { + bins += currentBin.toVector + } + + bins.filter { bin => + bin.size > 1 || // bin has more than one file or + bin.size == 1 && optimizeContext.reorg.nonEmpty || // always rewrite files during reorg + isMultiDimClustering // multi-clustering + }.map(b => (partition, b)) + } + } + + /** + * Utility method to run a Spark job to compact the files in given bin + * + * @param txn [[OptimisticTransaction]] instance in use to commit the changes to DeltaLog. + * @param partition Partition values of the partition that files in [[bin]] belongs to. + * @param bin List of files to compact into one large file. + * @param maxFileSize Targeted output file size in bytes + */ + private def runOptimizeBinJob( + txn: OptimisticTransaction, + partition: Map[String, String], + bin: Seq[AddFile], + maxFileSize: Long): Seq[FileAction] = { + val baseTablePath = txn.deltaLog.dataPath + + var input = txn.deltaLog.createDataFrame(txn.snapshot, bin, actionTypeOpt = Some("Optimize")) + input = RowTracking.preserveRowTrackingColumns(input, txn.snapshot) + val repartitionDF = if (isMultiDimClustering) { + val totalSize = bin.map(_.size).sum + val approxNumFiles = Math.max(1, totalSize / maxFileSize).toInt + MultiDimClustering.cluster( + input, + approxNumFiles, + clusteringColumns, + optimizeStrategy.curve) + } else { + val useRepartition = sparkSession.sessionState.conf.getConf( + DeltaSQLConf.DELTA_OPTIMIZE_REPARTITION_ENABLED) + if (useRepartition) { + input.repartition(numPartitions = 1) + } else { + input.coalesce(numPartitions = 1) + } + } + + val partitionDesc = partition.toSeq.map(entry => entry._1 + "=" + entry._2).mkString(",") + + val partitionName = if (partition.isEmpty) "" else s" in partition ($partitionDesc)" + val description = s"$baseTablePath
Optimizing ${bin.size} files" + partitionName + sparkSession.sparkContext.setJobGroup( + sparkSession.sparkContext.getLocalProperty(SPARK_JOB_GROUP_ID), + description) + + val binInfo = optimizeStrategy.initNewBin + val addFiles = txn.writeFiles(repartitionDF, None, isOptimize = true, Nil).collect { + case a: AddFile => optimizeStrategy.tagAddFile(a, binInfo) + case other => + throw new IllegalStateException( + s"Unexpected action $other with type ${other.getClass}. File compaction job output" + + s"should only have AddFiles") + } + val removeFiles = bin.map(f => f.removeWithTimestamp(operationTimestamp, dataChange = false)) + val updates = addFiles ++ removeFiles + updates + } + + /** + * Attempts to commit the given actions to the log. In the case of a concurrent update, + * the given function will be invoked with a new transaction to allow custom conflict + * detection logic to indicate it is safe to try again, by returning `true`. + * + * This function will continue to try to commit to the log as long as `f` returns `true`, + * otherwise throws a subclass of [[ConcurrentModificationException]]. + */ + private def commitAndRetry( + txn: OptimisticTransaction, + optimizeOperation: Operation, + actions: Seq[Action], + metrics: Map[String, SQLMetric])(f: OptimisticTransaction => Boolean): Unit = { + try { + txn.registerSQLMetrics(sparkSession, metrics) + txn.commit(actions, optimizeOperation, + RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot)) + } catch { + case e: ConcurrentModificationException => + val newTxn = txn.deltaLog.startTransaction(txn.catalogTable) + if (f(newTxn)) { + logInfo("Retrying commit after checking for semantic conflicts with concurrent updates.") + commitAndRetry(newTxn, optimizeOperation, actions, metrics)(f) + } else { + logWarning("Semantic conflicts detected. Aborting operation.") + throw e + } + } + } + + /** Create the appropriate [[Operation]] object for txn commit history */ + private def getOperation(): Operation = { + if (optimizeContext.reorg.nonEmpty) { + DeltaOperations.Reorg(partitionPredicate) + } else { + DeltaOperations.Optimize( + predicate = partitionPredicate, + zOrderBy = zOrderByColumns, + auto = isAutoCompact, + clusterBy = if (isClusteredTable) Option(clusteringColumns).filter(_.nonEmpty) else None) + } + } + + /** Create a map of SQL metrics for adding to the commit history. */ + private def createMetrics( + sparkContext: SparkContext, + addedFiles: Seq[AddFile], + removedFiles: Seq[RemoveFile], + removedDVs: Seq[DeletionVectorDescriptor]): Map[String, SQLMetric] = { + + def setAndReturnMetric(description: String, value: Long) = { + val metric = createMetric(sparkContext, description) + metric.set(value) + metric + } + + def totalSize(actions: Seq[FileAction]): Long = { + var totalSize = 0L + actions.foreach { file => + val fileSize = file match { + case addFile: AddFile => addFile.size + case removeFile: RemoveFile => removeFile.size.getOrElse(0L) + case default => + throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") + } + totalSize += fileSize + } + totalSize + } + + val (deletionVectorRowsRemoved, deletionVectorBytesRemoved) = + removedDVs.map(dv => (dv.cardinality, dv.sizeInBytes.toLong)) + .reduceLeftOption((dv1, dv2) => (dv1._1 + dv2._1, dv1._2 + dv2._2)) + .getOrElse((0L, 0L)) + + val dvMetrics: Map[String, SQLMetric] = Map( + "numDeletionVectorsRemoved" -> + setAndReturnMetric( + "total number of deletion vectors removed", + removedDVs.size), + "numDeletionVectorRowsRemoved" -> + setAndReturnMetric( + "total number of deletion vector rows removed", + deletionVectorRowsRemoved), + "numDeletionVectorBytesRemoved" -> + setAndReturnMetric( + "total number of bytes of removed deletion vectors", + deletionVectorBytesRemoved)) + + val sizeStats = FileSizeStatsWithHistogram.create(addedFiles.map(_.size).sorted) + Map[String, SQLMetric]( + "minFileSize" -> setAndReturnMetric("minimum file size", sizeStats.get.min), + "p25FileSize" -> setAndReturnMetric("25th percentile file size", sizeStats.get.p25), + "p50FileSize" -> setAndReturnMetric("50th percentile file size", sizeStats.get.p50), + "p75FileSize" -> setAndReturnMetric("75th percentile file size", sizeStats.get.p75), + "maxFileSize" -> setAndReturnMetric("maximum file size", sizeStats.get.max), + "numAddedFiles" -> setAndReturnMetric("total number of files added.", addedFiles.size), + "numRemovedFiles" -> setAndReturnMetric("total number of files removed.", removedFiles.size), + "numAddedBytes" -> setAndReturnMetric("total number of bytes added", totalSize(addedFiles)), + "numRemovedBytes" -> + setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)) + ) ++ dvMetrics + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala new file mode 100644 index 000000000000..7b4c3231b8c3 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala @@ -0,0 +1,345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta.commands + +import org.apache.gluten.expression.ConverterUtils + +import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException} +import org.apache.spark.internal.Logging +import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage +import org.apache.spark.internal.io.SparkHadoopWriterUtils +import org.apache.spark.shuffle.FetchFailedException +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.types.DataTypeUtils +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddFile, FileAction} +import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2} +import org.apache.spark.sql.errors.QueryExecutionErrors +import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper +import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.v1.clickhouse._ +import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StructType +import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} + +import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} +import org.apache.hadoop.mapreduce.{TaskAttemptContext, TaskAttemptID, TaskID, TaskType} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl + +import java.util.{Date, UUID} + +import scala.collection.mutable.ArrayBuffer + +object OptimizeTableCommandOverwrites extends Logging { + + case class TaskDescription( + path: String, + database: String, + tableName: String, + snapshotId: String, + orderByKeyOption: Option[Seq[String]], + lowCardKeyOption: Option[Seq[String]], + minmaxIndexKeyOption: Option[Seq[String]], + bfIndexKeyOption: Option[Seq[String]], + setIndexKeyOption: Option[Seq[String]], + primaryKeyOption: Option[Seq[String]], + partitionColumns: Seq[String], + partList: Seq[String], + tableSchema: StructType, + clickhouseTableConfigs: Map[String, String], + serializableHadoopConf: SerializableConfiguration, + jobIdInstant: Long, + partitionDir: Option[String], + bucketDir: Option[String] + ) + + private def executeTask( + description: TaskDescription, + sparkStageId: Int, + sparkPartitionId: Int, + sparkAttemptNumber: Int + ): MergeTreeWriteTaskResult = { + + val jobId = SparkHadoopWriterUtils.createJobID(new Date(description.jobIdInstant), sparkStageId) + val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId) + val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber) + + // Set up the attempt context required to use in the output committer. + val taskAttemptContext: TaskAttemptContext = { + // Set up the configuration object + val hadoopConf = description.serializableHadoopConf.value + hadoopConf.set("mapreduce.job.id", jobId.toString) + hadoopConf.set("mapreduce.task.id", taskAttemptId.getTaskID.toString) + hadoopConf.set("mapreduce.task.attempt.id", taskAttemptId.toString) + hadoopConf.setBoolean("mapreduce.task.ismap", true) + hadoopConf.setInt("mapreduce.task.partition", 0) + + new TaskAttemptContextImpl(hadoopConf, taskAttemptId) + } + + try { + Utils.tryWithSafeFinallyAndFailureCallbacks(block = { + + val uuid = UUID.randomUUID.toString + + val planWithSplitInfo = CHMergeTreeWriterInjects.genMergeTreeWriteRel( + description.path, + description.database, + description.tableName, + description.snapshotId, + description.orderByKeyOption, + description.lowCardKeyOption, + description.minmaxIndexKeyOption, + description.bfIndexKeyOption, + description.setIndexKeyOption, + description.primaryKeyOption, + description.partitionColumns, + description.partList, + ConverterUtils.convertNamedStructJson(description.tableSchema), + description.clickhouseTableConfigs, + DataTypeUtils.toAttributes(description.tableSchema) + ) + + val datasourceJniWrapper = new CHDatasourceJniWrapper() + val returnedMetrics = + datasourceJniWrapper.nativeMergeMTParts( + planWithSplitInfo.plan, + planWithSplitInfo.splitInfo, + uuid, + taskId.getId.toString, + description.partitionDir.getOrElse(""), + description.bucketDir.getOrElse("") + ) + if (returnedMetrics != null && returnedMetrics.nonEmpty) { + val addFiles = AddFileTags.partsMetricsToAddFile( + description.database, + description.tableName, + description.path, + returnedMetrics, + Seq(Utils.localHostName())) + + val (taskCommitMessage, taskCommitTime) = Utils.timeTakenMs { + // committer.commitTask(taskAttemptContext) + new TaskCommitMessage(addFiles.toSeq) + } + +// val summary = MergeTreeExecutedWriteSummary( +// updatedPartitions = updatedPartitions.toSet, +// stats = statsTrackers.map(_.getFinalStats(taskCommitTime))) + MergeTreeWriteTaskResult(taskCommitMessage, null) + } else { + throw new IllegalStateException() + } + })( + catchBlock = { + // If there is an error, abort the task + logError(s"Job $jobId aborted.") + }, + finallyBlock = {}) + } catch { + case e: FetchFailedException => + throw e + case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput => + // If any output file to write already exists, it does not make sense to re-run this task. + // We throw the exception and let Executor throw ExceptionFailure to abort the job. + throw new TaskOutputFileAlreadyExistException(f) + case t: Throwable => + throw QueryExecutionErrors.taskFailedWhileWritingRowsError(description.path, t) + } + + } + + def runOptimizeBinJobClickhouse( + txn: OptimisticTransaction, + partitionValues: Map[String, String], + bucketNum: String, + bin: Seq[AddFile], + maxFileSize: Long): Seq[FileAction] = { + val tableV2 = ClickHouseTableV2.getTable(txn.deltaLog); + + val sparkSession = SparkSession.getActiveSession.get + + val rddWithNonEmptyPartitions = + sparkSession.sparkContext.parallelize(Array.empty[InternalRow], 1) + + val jobIdInstant = new Date().getTime + val ret = new Array[MergeTreeWriteTaskResult](rddWithNonEmptyPartitions.partitions.length) + + val serializableHadoopConf = new SerializableConfiguration( + sparkSession.sessionState.newHadoopConfWithOptions( + txn.metadata.configuration ++ txn.deltaLog.options)) + + val partitionDir = if (tableV2.partitionColumns.isEmpty) { + None + } else { + Some(tableV2.partitionColumns.map(c => c + "=" + partitionValues(c)).mkString("/")) + } + + val bucketDir = if (tableV2.bucketOption.isEmpty) { + None + } else { + Some(bucketNum) + } + + val description = TaskDescription.apply( + txn.deltaLog.dataPath.toString, + tableV2.dataBaseName, + tableV2.tableName, + ClickhouseSnapshot.genSnapshotId(tableV2.initialSnapshot), + tableV2.orderByKeyOption, + tableV2.lowCardKeyOption, + tableV2.minmaxIndexKeyOption, + tableV2.bfIndexKeyOption, + tableV2.setIndexKeyOption, + tableV2.primaryKeyOption, + tableV2.partitionColumns, + bin.map(_.asInstanceOf[AddMergeTreeParts].name), + tableV2.schema(), + tableV2.clickhouseTableConfigs, + serializableHadoopConf, + jobIdInstant, + partitionDir, + bucketDir + ) + sparkSession.sparkContext.runJob( + rddWithNonEmptyPartitions, + (taskContext: TaskContext, _: Iterator[InternalRow]) => { + executeTask( + description, + taskContext.stageId(), + taskContext.partitionId(), + taskContext.taskAttemptId().toInt & Integer.MAX_VALUE + ) + }, + rddWithNonEmptyPartitions.partitions.indices, + (index, res: MergeTreeWriteTaskResult) => { + ret(index) = res + } + ) + + val addFiles = ret + .flatMap(_.commitMsg.obj.asInstanceOf[Seq[AddFile]]) + .toSeq + + val removeFiles = + bin.map(f => f.removeWithTimestamp(new SystemClock().getTimeMillis(), dataChange = false)) + addFiles ++ removeFiles + + } + + def getDeltaLogClickhouse( + spark: SparkSession, + path: Option[String], + tableIdentifier: Option[TableIdentifier], + operationName: String, + hadoopConf: Map[String, String] = Map.empty): DeltaLog = { + val tablePath = + if (path.nonEmpty) { + new Path(path.get) + } else if (tableIdentifier.nonEmpty) { + val sessionCatalog = spark.sessionState.catalog + lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get) + + if (CHDataSourceUtils.isClickhousePath(spark, tableIdentifier.get)) { + new Path(tableIdentifier.get.table) + } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) { + new Path(metadata.location) + } else { + DeltaTableIdentifier(spark, tableIdentifier.get) match { + case Some(id) if id.path.nonEmpty => + new Path(id.path.get) + case Some(id) if id.table.nonEmpty => + new Path(metadata.location) + case _ => + if (metadata.tableType == CatalogTableType.VIEW) { + throw DeltaErrors.viewNotSupported(operationName) + } + throw DeltaErrors.notADeltaTableException(operationName) + } + } + } else { + throw DeltaErrors.missingTableIdentifierException(operationName) + } + + val startTime = Some(System.currentTimeMillis) + val deltaLog = DeltaLog.forTable(spark, tablePath, hadoopConf) + if (deltaLog.update(checkIfUpdatedSinceTs = startTime).version < 0) { + throw DeltaErrors.notADeltaTableException( + operationName, + DeltaTableIdentifier(path, tableIdentifier)) + } + deltaLog + } + + /** + * Extracts the [[DeltaTableV2]] from a LogicalPlan iff the LogicalPlan is a [[ResolvedTable]] + * with either a [[DeltaTableV2]] or a [[V1Table]] that is referencing a Delta table. In all + * other cases this method will throw a "Table not found" exception. + */ + def getDeltaTable(target: LogicalPlan, cmd: String): DeltaTableV2 = { + // TODO: Remove this wrapper and let former callers invoke DeltaTableV2.extractFrom directly. + ClickHouseTableV2.extractFrom(target, cmd) + } + + def groupFilesIntoBinsClickhouse( + partitionsToCompact: Seq[((String, Map[String, String]), Seq[AddFile])], + maxTargetFileSize: Long): Seq[((String, Map[String, String]), Seq[AddFile])] = { + partitionsToCompact.flatMap { + case (partition, files) => + val bins = new ArrayBuffer[Seq[AddFile]]() + + val currentBin = new ArrayBuffer[AddFile]() + var currentBinSize = 0L + + files.sortBy(_.size).foreach { + file => + // Generally, a bin is a group of existing files, whose total size does not exceed the + // desired maxFileSize. They will be coalesced into a single output file. + // However, if isMultiDimClustering = true, all files in a partition will be read by the + // same job, the data will be range-partitioned and + // numFiles = totalFileSize / maxFileSize + // will be produced. See below. + + // isMultiDimClustering is always false for Gluten Clickhouse for now + if (file.size + currentBinSize > maxTargetFileSize /* && !isMultiDimClustering */ ) { + bins += currentBin.toVector + currentBin.clear() + currentBin += file + currentBinSize = file.size + } else { + currentBin += file + currentBinSize += file.size + } + } + + if (currentBin.nonEmpty) { + bins += currentBin.toVector + } + + bins + .map(b => (partition, b)) + // select bins that have at least two files or in case of multi-dim clustering + // select all bins + .filter(_._2.size > 1 /* || isMultiDimClustering */ ) + } + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala new file mode 100644 index 000000000000..9a7fb96775f0 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala @@ -0,0 +1,556 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands + +// scalastyle:off import.ordering.noEmptyLine +import java.util.concurrent.TimeUnit + +import org.apache.spark.sql.delta.metric.IncrementMetric +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} +import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC, CDC_TYPE_UPDATE_POSTIMAGE, CDC_TYPE_UPDATE_PREIMAGE} +import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.hadoop.fs.Path + +import org.apache.spark.SparkContext +import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, If, Literal} +import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral +import org.apache.spark.sql.catalyst.plans.QueryPlan +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.command.LeafRunnableCommand +import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.metric.SQLMetrics.{createMetric, createTimingMetric} +import org.apache.spark.sql.functions.{array, col, explode, input_file_name, lit, split, struct} +import org.apache.spark.sql.types.LongType + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0. It is modified to overcome the following issues: + * 1. In Clickhouse backend, we can't implement input_file_name() correctly, we can only implement + * it so that it return a a list of filenames (concated by ','). + */ + +/** + * Performs an Update using `updateExpression` on the rows that match `condition` + * + * Algorithm: + * 1) Identify the affected files, i.e., the files that may have the rows to be updated. + * 2) Scan affected files, apply the updates, and generate a new DF with updated rows. + * 3) Use the Delta protocol to atomically write the new DF as new files and remove + * the affected files that are identified in step 1. + */ +case class UpdateCommand( + tahoeFileIndex: TahoeFileIndex, + catalogTable: Option[CatalogTable], + target: LogicalPlan, + updateExpressions: Seq[Expression], + condition: Option[Expression]) + extends LeafRunnableCommand with DeltaCommand { + + override val output: Seq[Attribute] = { + Seq(AttributeReference("num_affected_rows", LongType)()) + } + + override def innerChildren: Seq[QueryPlan[_]] = Seq(target) + + @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() + + override lazy val metrics = Map[String, SQLMetric]( + "numAddedFiles" -> createMetric(sc, "number of files added."), + "numAddedBytes" -> createMetric(sc, "number of bytes added"), + "numRemovedFiles" -> createMetric(sc, "number of files removed."), + "numRemovedBytes" -> createMetric(sc, "number of bytes removed"), + "numUpdatedRows" -> createMetric(sc, "number of rows updated."), + "numCopiedRows" -> createMetric(sc, "number of rows copied."), + "executionTimeMs" -> + createTimingMetric(sc, "time taken to execute the entire operation"), + "scanTimeMs" -> + createTimingMetric(sc, "time taken to scan the files for matches"), + "rewriteTimeMs" -> + createTimingMetric(sc, "time taken to rewrite the matched files"), + "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), + "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), + "numTouchedRows" -> createMetric(sc, "number of rows touched (copied + updated)"), + "numDeletionVectorsAdded" -> createMetric(sc, "number of deletion vectors added"), + "numDeletionVectorsRemoved" -> createMetric(sc, "number of deletion vectors removed"), + "numDeletionVectorsUpdated" -> createMetric(sc, "number of deletion vectors updated") + ) + + final override def run(sparkSession: SparkSession): Seq[Row] = { + recordDeltaOperation(tahoeFileIndex.deltaLog, "delta.dml.update") { + val deltaLog = tahoeFileIndex.deltaLog + deltaLog.withNewTransaction(catalogTable) { txn => + DeltaLog.assertRemovable(txn.snapshot) + if (hasBeenExecuted(txn, sparkSession)) { + sendDriverMetrics(sparkSession, metrics) + return Seq.empty + } + performUpdate(sparkSession, deltaLog, txn) + } + // Re-cache all cached plans(including this relation itself, if it's cached) that refer to + // this data source relation. + sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) + } + Seq(Row(metrics("numUpdatedRows").value)) + } + + private def performUpdate( + sparkSession: SparkSession, deltaLog: DeltaLog, txn: OptimisticTransaction): Unit = { + import org.apache.spark.sql.delta.implicits._ + + var numTouchedFiles: Long = 0 + var numRewrittenFiles: Long = 0 + var numAddedBytes: Long = 0 + var numRemovedBytes: Long = 0 + var numAddedChangeFiles: Long = 0 + var changeFileBytes: Long = 0 + var scanTimeMs: Long = 0 + var rewriteTimeMs: Long = 0 + var numDeletionVectorsAdded: Long = 0 + var numDeletionVectorsRemoved: Long = 0 + var numDeletionVectorsUpdated: Long = 0 + + val startTime = System.nanoTime() + val numFilesTotal = txn.snapshot.numOfFiles + + val updateCondition = condition.getOrElse(Literal.TrueLiteral) + val (metadataPredicates, dataPredicates) = + DeltaTableUtils.splitMetadataAndDataPredicates( + updateCondition, txn.metadata.partitionColumns, sparkSession) + + // Should we write the DVs to represent updated rows? + val shouldWriteDeletionVectors = shouldWritePersistentDeletionVectors(sparkSession, txn) + val candidateFiles = txn.filterFiles( + metadataPredicates ++ dataPredicates, + keepNumRecords = shouldWriteDeletionVectors) + + val nameToAddFile = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) + + scanTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) + + val filesToRewrite: Seq[TouchedFileWithDV] = if (candidateFiles.isEmpty) { + // Case 1: Do nothing if no row qualifies the partition predicates + // that are part of Update condition + Nil + } else if (dataPredicates.isEmpty) { + // Case 2: Update all the rows from the files that are in the specified partitions + // when the data filter is empty + candidateFiles + .map(f => TouchedFileWithDV(f.path, f, newDeletionVector = null, deletedRows = 0L)) + } else { + // Case 3: Find all the affected files using the user-specified condition + val fileIndex = new TahoeBatchFileIndex( + sparkSession, "update", candidateFiles, deltaLog, tahoeFileIndex.path, txn.snapshot) + + val touchedFilesWithDV = if (shouldWriteDeletionVectors) { + // Case 3.1: Find all the affected files via DV path + val targetDf = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( + sparkSession, + target, + fileIndex) + + // Does the target table already has DVs enabled? If so, we need to read the table + // with deletion vectors. + val mustReadDeletionVectors = DeletionVectorUtils.deletionVectorsReadable(txn.snapshot) + + DMLWithDeletionVectorsHelper.findTouchedFiles( + sparkSession, + txn, + mustReadDeletionVectors, + deltaLog, + targetDf, + fileIndex, + updateCondition, + opName = "UPDATE") + } else { + // Case 3.2: Find all the affected files using the non-DV path + // Keep everything from the resolved target except a new TahoeFileIndex + // that only involves the affected files instead of all files. + val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) + val data = Dataset.ofRows(sparkSession, newTarget) + val incrUpdatedCountExpr = IncrementMetric(TrueLiteral, metrics("numUpdatedRows")) + val pathsToRewrite = + withStatusCode("DELTA", UpdateCommand.FINDING_TOUCHED_FILES_MSG) { + // --- modified start + data.filter(new Column(updateCondition)) + .select(input_file_name().as("input_files")) + .filter(new Column(incrUpdatedCountExpr)) + .select(explode(split(col("input_files"), ","))) + .distinct() + .as[String] + .collect() + // --- modified end + } + + // Wrap AddFile into TouchedFileWithDV that has empty DV. + pathsToRewrite + .map(getTouchedFile(deltaLog.dataPath, _, nameToAddFile)) + .map(f => TouchedFileWithDV(f.path, f, newDeletionVector = null, deletedRows = 0L)) + .toSeq + } + // Refresh scan time for Case 3, since we performed scan here. + scanTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime) + touchedFilesWithDV + } + + val totalActions = { + // When DV is on, we first mask removed rows with DVs and generate (remove, add) pairs. + val actionsForExistingFiles = if (shouldWriteDeletionVectors) { + // When there's no data predicate, all matched files are removed. + if (dataPredicates.isEmpty) { + val operationTimestamp = System.currentTimeMillis() + filesToRewrite.map(_.fileLogEntry.removeWithTimestamp(operationTimestamp)) + } else { + // When there is data predicate, we generate (remove, add) pairs. + val filesToRewriteWithDV = filesToRewrite.filter(_.newDeletionVector != null) + val (dvActions, metricMap) = DMLWithDeletionVectorsHelper.processUnmodifiedData( + sparkSession, + filesToRewriteWithDV, + txn.snapshot) + metrics("numUpdatedRows").set(metricMap("numModifiedRows")) + numDeletionVectorsAdded = metricMap("numDeletionVectorsAdded") + numDeletionVectorsRemoved = metricMap("numDeletionVectorsRemoved") + numDeletionVectorsUpdated = metricMap("numDeletionVectorsUpdated") + numTouchedFiles = metricMap("numRemovedFiles") + dvActions + } + } else { + // Without DV we'll leave the job to `rewriteFiles`. + Nil + } + + // When DV is on, we write out updated rows only. The return value will be only `add` actions. + // When DV is off, we write out updated rows plus unmodified rows from the same file, then + // return `add` and `remove` actions. + val rewriteStartNs = System.nanoTime() + val actionsForNewFiles = + withStatusCode("DELTA", UpdateCommand.rewritingFilesMsg(filesToRewrite.size)) { + if (filesToRewrite.nonEmpty) { + rewriteFiles( + sparkSession, + txn, + rootPath = tahoeFileIndex.path, + inputLeafFiles = filesToRewrite.map(_.fileLogEntry), + nameToAddFileMap = nameToAddFile, + condition = updateCondition, + generateRemoveFileActions = !shouldWriteDeletionVectors, + copyUnmodifiedRows = !shouldWriteDeletionVectors) + } else { + Nil + } + } + rewriteTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - rewriteStartNs) + + numTouchedFiles = filesToRewrite.length + val (addActions, removeActions) = actionsForNewFiles.partition(_.isInstanceOf[AddFile]) + numRewrittenFiles = addActions.size + numAddedBytes = addActions.map(_.getFileSize).sum + numRemovedBytes = removeActions.map(_.getFileSize).sum + + actionsForExistingFiles ++ actionsForNewFiles + } + + val changeActions = totalActions.collect { case f: AddCDCFile => f } + numAddedChangeFiles = changeActions.size + changeFileBytes = changeActions.map(_.size).sum + + metrics("numAddedFiles").set(numRewrittenFiles) + metrics("numAddedBytes").set(numAddedBytes) + metrics("numAddedChangeFiles").set(numAddedChangeFiles) + metrics("changeFileBytes").set(changeFileBytes) + metrics("numRemovedFiles").set(numTouchedFiles) + metrics("numRemovedBytes").set(numRemovedBytes) + metrics("executionTimeMs").set(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)) + metrics("scanTimeMs").set(scanTimeMs) + metrics("rewriteTimeMs").set(rewriteTimeMs) + // In the case where the numUpdatedRows is not captured, we can siphon out the metrics from + // the BasicWriteStatsTracker. This is for case 2 where the update condition contains only + // metadata predicates and so the entire partition is re-written. + val outputRows = txn.getMetric("numOutputRows").map(_.value).getOrElse(-1L) + if (metrics("numUpdatedRows").value == 0 && outputRows != 0 && + metrics("numCopiedRows").value == 0) { + // We know that numTouchedRows = numCopiedRows + numUpdatedRows. + // Since an entire partition was re-written, no rows were copied. + // So numTouchedRows == numUpdateRows + metrics("numUpdatedRows").set(metrics("numTouchedRows").value) + } else { + // This is for case 3 where the update condition contains both metadata and data predicates + // so relevant files will have some rows updated and some rows copied. We don't need to + // consider case 1 here, where no files match the update condition, as we know that + // `totalActions` is empty. + metrics("numCopiedRows").set( + metrics("numTouchedRows").value - metrics("numUpdatedRows").value) + metrics("numDeletionVectorsAdded").set(numDeletionVectorsAdded) + metrics("numDeletionVectorsRemoved").set(numDeletionVectorsRemoved) + metrics("numDeletionVectorsUpdated").set(numDeletionVectorsUpdated) + } + txn.registerSQLMetrics(sparkSession, metrics) + + val finalActions = createSetTransaction(sparkSession, deltaLog).toSeq ++ totalActions + txn.commitIfNeeded( + actions = finalActions, + op = DeltaOperations.Update(condition), + tags = RowTracking.addPreservedRowTrackingTagIfNotSet(txn.snapshot)) + sendDriverMetrics(sparkSession, metrics) + + recordDeltaEvent( + deltaLog, + "delta.dml.update.stats", + data = UpdateMetric( + condition = condition.map(_.sql).getOrElse("true"), + numFilesTotal, + numTouchedFiles, + numRewrittenFiles, + numAddedChangeFiles, + changeFileBytes, + scanTimeMs, + rewriteTimeMs, + numDeletionVectorsAdded, + numDeletionVectorsRemoved, + numDeletionVectorsUpdated) + ) + } + + /** + * Scan all the affected files and write out the updated files. + * + * When CDF is enabled, includes the generation of CDC preimage and postimage columns for + * changed rows. + * + * @return a list of [[FileAction]]s, consisting of newly-written data and CDC files and old + * files that have been removed. + */ + private def rewriteFiles( + spark: SparkSession, + txn: OptimisticTransaction, + rootPath: Path, + inputLeafFiles: Seq[AddFile], + nameToAddFileMap: Map[String, AddFile], + condition: Expression, + generateRemoveFileActions: Boolean, + copyUnmodifiedRows: Boolean): Seq[FileAction] = { + // Number of total rows that we have seen, i.e. are either copying or updating (sum of both). + // This will be used later, along with numUpdatedRows, to determine numCopiedRows. + val incrTouchedCountExpr = IncrementMetric(TrueLiteral, metrics("numTouchedRows")) + + // Containing the map from the relative file path to AddFile + val baseRelation = buildBaseRelation( + spark, txn, "update", rootPath, inputLeafFiles.map(_.path), nameToAddFileMap) + val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) + val (targetDf, finalOutput, finalUpdateExpressions) = UpdateCommand.preserveRowTrackingColumns( + targetDfWithoutRowTrackingColumns = Dataset.ofRows(spark, newTarget), + snapshot = txn.snapshot, + targetOutput = target.output, + updateExpressions) + + val targetDfWithEvaluatedCondition = { + val evalDf = targetDf.withColumn(UpdateCommand.CONDITION_COLUMN_NAME, new Column(condition)) + val copyAndUpdateRowsDf = if (copyUnmodifiedRows) { + evalDf + } else { + evalDf.filter(new Column(UpdateCommand.CONDITION_COLUMN_NAME)) + } + copyAndUpdateRowsDf.filter(new Column(incrTouchedCountExpr)) + } + + val updatedDataFrame = UpdateCommand.withUpdatedColumns( + finalOutput, + finalUpdateExpressions, + condition, + targetDfWithEvaluatedCondition, + UpdateCommand.shouldOutputCdc(txn)) + + val addFiles = txn.writeFiles(updatedDataFrame) + + val removeFiles = if (generateRemoveFileActions) { + val operationTimestamp = System.currentTimeMillis() + inputLeafFiles.map(_.removeWithTimestamp(operationTimestamp)) + } else { + Nil + } + + addFiles ++ removeFiles + } + + def shouldWritePersistentDeletionVectors( + spark: SparkSession, txn: OptimisticTransaction): Boolean = { + spark.conf.get(DeltaSQLConf.UPDATE_USE_PERSISTENT_DELETION_VECTORS) && + DeletionVectorUtils.deletionVectorsWritable(txn.snapshot) + } +} + +object UpdateCommand { + val FILE_NAME_COLUMN = "_input_file_name_" + val CONDITION_COLUMN_NAME = "__condition__" + val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for UPDATE operation" + + def rewritingFilesMsg(numFilesToRewrite: Long): String = + s"Rewriting $numFilesToRewrite files for UPDATE operation" + + /** + * Whether or not CDC is enabled on this table and, thus, if we should output CDC data during this + * UPDATE operation. + */ + def shouldOutputCdc(txn: OptimisticTransaction): Boolean = { + DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) + } + + /** + * Build the new columns. If the condition matches, generate the new value using + * the corresponding UPDATE EXPRESSION; otherwise, keep the original column value. + * + * When CDC is enabled, includes the generation of CDC pre-image and post-image columns for + * changed rows. + * + * @param originalExpressions the original column values + * @param updateExpressions the update transformation to perform on the input DataFrame + * @param dfWithEvaluatedCondition source DataFrame on which we will apply the update expressions + * with an additional column CONDITION_COLUMN_NAME which is the + * true/false value of if the update condition is satisfied + * @param condition update condition + * @param shouldOutputCdc if we should output CDC data during this UPDATE operation. + * @return the updated DataFrame, with extra CDC columns if CDC is enabled + */ + def withUpdatedColumns( + originalExpressions: Seq[Attribute], + updateExpressions: Seq[Expression], + condition: Expression, + dfWithEvaluatedCondition: DataFrame, + shouldOutputCdc: Boolean): DataFrame = { + val resultDf = if (shouldOutputCdc) { + val namedUpdateCols = updateExpressions.zip(originalExpressions).map { + case (expr, targetCol) => new Column(expr).as(targetCol.name, targetCol.metadata) + } + + // Build an array of output rows to be unpacked later. If the condition is matched, we + // generate CDC pre and postimages in addition to the final output row; if the condition + // isn't matched, we just generate a rewritten no-op row without any CDC events. + val preimageCols = originalExpressions.map(new Column(_)) :+ + lit(CDC_TYPE_UPDATE_PREIMAGE).as(CDC_TYPE_COLUMN_NAME) + val postimageCols = namedUpdateCols :+ + lit(CDC_TYPE_UPDATE_POSTIMAGE).as(CDC_TYPE_COLUMN_NAME) + val notCdcCol = new Column(CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME) + val updatedDataCols = namedUpdateCols :+ notCdcCol + val noopRewriteCols = originalExpressions.map(new Column(_)) :+ notCdcCol + val packedUpdates = array( + struct(preimageCols: _*), + struct(postimageCols: _*), + struct(updatedDataCols: _*) + ).expr + + val packedData = if (condition == Literal.TrueLiteral) { + packedUpdates + } else { + If( + UnresolvedAttribute(CONDITION_COLUMN_NAME), + packedUpdates, // if it should be updated, then use `packagedUpdates` + array(struct(noopRewriteCols: _*)).expr) // else, this is a noop rewrite + } + + // Explode the packed array, and project back out the final data columns. + val finalColumns = (originalExpressions :+ UnresolvedAttribute(CDC_TYPE_COLUMN_NAME)).map { + a => col(s"packedData.`${a.name}`").as(a.name, a.metadata) + } + dfWithEvaluatedCondition + .select(explode(new Column(packedData)).as("packedData")) + .select(finalColumns: _*) + } else { + val finalCols = updateExpressions.zip(originalExpressions).map { case (update, original) => + val updated = if (condition == Literal.TrueLiteral) { + update + } else { + If(UnresolvedAttribute(CONDITION_COLUMN_NAME), update, original) + } + new Column(updated).as(original.name, original.metadata) + } + + dfWithEvaluatedCondition.select(finalCols: _*) + } + + resultDf.drop(CONDITION_COLUMN_NAME) + } + + /** + * Preserve the row tracking columns when performing an UPDATE. + * + * @param targetDfWithoutRowTrackingColumns The target DataFrame on which the UPDATE + * operation is to be performed. + * @param snapshot Snapshot of the Delta table at the start of + * the transaction. + * @param targetOutput The output schema of the target DataFrame. + * @param updateExpressions The update transformation to perform on the + * target DataFrame. + * @return + * 1. targetDf: The target DataFrame that includes the preserved row tracking columns. + * 2. finalOutput: The final output schema, including the preserved row tracking columns. + * 3. finalUpdateExpressions: The final update expressions, including transformations + * for the preserved row tracking columns. + */ + def preserveRowTrackingColumns( + targetDfWithoutRowTrackingColumns: DataFrame, + snapshot: Snapshot, + targetOutput: Seq[Attribute] = Seq.empty, + updateExpressions: Seq[Expression] = Seq.empty): + (DataFrame, Seq[Attribute], Seq[Expression]) = { + val targetDf = RowTracking.preserveRowTrackingColumns( + targetDfWithoutRowTrackingColumns, snapshot) + + val rowIdAttributeOpt = MaterializedRowId.getAttribute(snapshot, targetDf) + val rowCommitVersionAttributeOpt = + MaterializedRowCommitVersion.getAttribute(snapshot, targetDf) + val finalOutput = targetOutput ++ rowIdAttributeOpt ++ rowCommitVersionAttributeOpt + + val finalUpdateExpressions = updateExpressions ++ + rowIdAttributeOpt ++ + rowCommitVersionAttributeOpt.map(_ => Literal(null, LongType)) + (targetDf, finalOutput, finalUpdateExpressions) + } +} + +/** + * Used to report details about update. + * + * @param condition: what was the update condition + * @param numFilesTotal: how big is the table + * @param numTouchedFiles: how many files did we touch + * @param numRewrittenFiles: how many files had to be rewritten + * @param numAddedChangeFiles: how many change files were generated + * @param changeFileBytes: total size of change files generated + * @param scanTimeMs: how long did finding take + * @param rewriteTimeMs: how long did rewriting take + * + * @note All the time units are milliseconds. + */ +case class UpdateMetric( + condition: String, + numFilesTotal: Long, + numTouchedFiles: Long, + numRewrittenFiles: Long, + numAddedChangeFiles: Long, + changeFileBytes: Long, + scanTimeMs: Long, + rewriteTimeMs: Long, + numDeletionVectorsAdded: Long, + numDeletionVectorsRemoved: Long, + numDeletionVectorsUpdated: Long +) diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala new file mode 100644 index 000000000000..987a7c35fa8b --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -0,0 +1,735 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands + +// scalastyle:off import.ordering.noEmptyLine +import java.net.URI +import java.util.Date +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddFile, FileAction, RemoveFile} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.DeltaFileOperations +import org.apache.spark.sql.delta.util.DeltaFileOperations.tryDeleteNonRecursive +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + +import org.apache.gluten.utils.QueryPlanSelector +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.{Column, DataFrame, Dataset, Encoder, SparkSession} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType} +import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0. It is modified to overcome the following issues: + * 1. In Gluten, part is a directory, but VacuumCommand assumes part is a file. So we need some + * modifications to make it work. + * 2. Set the 'gluten.enabledForCurrentThread' to false, now gluten can not support vacuum cmd. + */ + +/** + * Vacuums the table by clearing all untracked files and folders within this table. + * First lists all the files and directories in the table, and gets the relative paths with + * respect to the base of the table. Then it gets the list of all tracked files for this table, + * which may or may not be within the table base path, and gets the relative paths of + * all the tracked files with respect to the base of the table. Files outside of the table path + * will be ignored. Then we take a diff of the files and delete directories that were already empty, + * and all files that are within the table that are no longer tracked. + */ +object VacuumCommand extends VacuumCommandImpl with Serializable { + + // --- modified start + case class FileNameAndSize(path: String, length: Long, isDir: Boolean = false) + // --- modified end + + /** + * path : fully qualified uri + * length: size in bytes + * isDir: boolean indicating if it is a directory + * modificationTime: file update time in milliseconds + */ + val INVENTORY_SCHEMA = StructType( + Seq( + StructField("path", StringType), + StructField("length", LongType), + StructField("isDir", BooleanType), + StructField("modificationTime", LongType) + )) + + /** + * Additional check on retention duration to prevent people from shooting themselves in the foot. + */ + protected def checkRetentionPeriodSafety( + spark: SparkSession, + retentionMs: Option[Long], + configuredRetention: Long): Unit = { + require(retentionMs.forall(_ >= 0), "Retention for Vacuum can't be less than 0.") + val checkEnabled = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) + val retentionSafe = retentionMs.forall(_ >= configuredRetention) + var configuredRetentionHours = TimeUnit.MILLISECONDS.toHours(configuredRetention) + if (TimeUnit.HOURS.toMillis(configuredRetentionHours) < configuredRetention) { + configuredRetentionHours += 1 + } + require(!checkEnabled || retentionSafe, + s"""Are you sure you would like to vacuum files with such a low retention period? If you have + |writers that are currently writing to this table, there is a risk that you may corrupt the + |state of your Delta table. + | + |If you are certain that there are no operations being performed on this table, such as + |insert/upsert/delete/optimize, then you may turn off this check by setting: + |spark.databricks.delta.retentionDurationCheck.enabled = false + | + |If you are not sure, please use a value not less than "$configuredRetentionHours hours". + """.stripMargin) + } + + /** + * Helper to compute all valid files based on basePath and Snapshot provided. + */ + private def getValidFilesFromSnapshot( + spark: SparkSession, + basePath: String, + snapshot: Snapshot, + retentionMillis: Option[Long], + hadoopConf: Broadcast[SerializableConfiguration], + clock: Clock, + checkAbsolutePathOnly: Boolean): DataFrame = { + import org.apache.spark.sql.delta.implicits._ + require(snapshot.version >= 0, "No state defined for this table. Is this really " + + "a Delta table? Refusing to garbage collect.") + + val snapshotTombstoneRetentionMillis = DeltaLog.tombstoneRetentionMillis(snapshot.metadata) + checkRetentionPeriodSafety(spark, retentionMillis, snapshotTombstoneRetentionMillis) + val deleteBeforeTimestamp = retentionMillis match { + case Some(millis) => clock.getTimeMillis() - millis + case _ => snapshot.minFileRetentionTimestamp + } + val relativizeIgnoreError = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) + + val canonicalizedBasePath = SparkPath.fromPathString(basePath).urlEncoded + snapshot.stateDS.mapPartitions { actions => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + actions.flatMap { + _.unwrap match { + // Existing tables may not store canonicalized paths, so we check both the canonicalized + // and non-canonicalized paths to ensure we don't accidentally delete wrong files. + case fa: FileAction if checkAbsolutePathOnly && + !fa.path.contains(basePath) && !fa.path.contains(canonicalizedBasePath) => Nil + case tombstone: RemoveFile if tombstone.delTimestamp < deleteBeforeTimestamp => Nil + case fa: FileAction => + getValidRelativePathsAndSubdirs( + fa, + fs, + reservoirBase, + relativizeIgnoreError + ) + case _ => Nil + } + } + }.toDF("path") + } + + def getFilesFromInventory(basePath: String, + partitionColumns: Seq[String], + inventory: DataFrame): Dataset[SerializableFileStatus] = { + implicit val fileNameAndSizeEncoder: Encoder[SerializableFileStatus] = + org.apache.spark.sql.Encoders.product[SerializableFileStatus] + + // filter out required fields from provided inventory DF + val inventorySchema = StructType( + inventory.schema.fields.filter(f => INVENTORY_SCHEMA.fields.map(_.name).contains(f.name)) + ) + if (inventorySchema != INVENTORY_SCHEMA) { + throw DeltaErrors.invalidInventorySchema(INVENTORY_SCHEMA.treeString) + } + + inventory + .filter(startswith(col("path"), lit(s"$basePath/"))) + .select( + substr(col("path"), lit(basePath.length + 2)).as("path"), + col("length"), col("isDir"), col("modificationTime") + ) + .flatMap { + row => + val path = row.getString(0) + if(!DeltaTableUtils.isHiddenDirectory(partitionColumns, path)) { + Seq(SerializableFileStatus(path, + row.getLong(1), row.getBoolean(2), row.getLong(3))) + } else { + None + } + } + } + + /** + * Clears all untracked files and folders within this table. If the inventory is not provided + * then the command first lists all the files and directories in the table, if inventory is + * provided then it will be used for identifying files and directories within the table and + * gets the relative paths with respect to the base of the table. Then the command gets the + * list of all tracked files for this table, which may or may not be within the table base path, + * and gets the relative paths of all the tracked files with respect to the base of the table. + * Files outside of the table path will be ignored. Then we take a diff of the files and delete + * directories that were already empty, and all files that are within the table that are no longer + * tracked. + * + * @param dryRun If set to true, no files will be deleted. Instead, we will list all files and + * directories that will be cleared. + * @param retentionHours An optional parameter to override the default Delta tombstone retention + * period + * @param inventory An optional dataframe of files and directories within the table generated + * from sources like blob store inventory report + * @return A Dataset containing the paths of the files/folders to delete in dryRun mode. Otherwise + * returns the base path of the table. + */ + def gc( + spark: SparkSession, + deltaLog: DeltaLog, + dryRun: Boolean = true, + retentionHours: Option[Double] = None, + inventory: Option[DataFrame] = None, + clock: Clock = new SystemClock): DataFrame = { + recordDeltaOperation(deltaLog, "delta.gc") { + + val vacuumStartTime = System.currentTimeMillis() + val path = deltaLog.dataPath + val deltaHadoopConf = deltaLog.newDeltaHadoopConf() + val fs = path.getFileSystem(deltaHadoopConf) + + import org.apache.spark.sql.delta.implicits._ + + val snapshot = deltaLog.update() + deltaLog.protocolWrite(snapshot.protocol) + + // --- modified start + val isMergeTreeFormat = ClickHouseConfig + .isMergeTreeFormatEngine(deltaLog.unsafeVolatileMetadata.configuration) + // --- modified end + + val snapshotTombstoneRetentionMillis = DeltaLog.tombstoneRetentionMillis(snapshot.metadata) + val retentionMillis = retentionHours.map(h => TimeUnit.HOURS.toMillis(math.round(h))) + val deleteBeforeTimestamp = retentionMillis match { + case Some(millis) => clock.getTimeMillis() - millis + case _ => snapshot.minFileRetentionTimestamp + } + // --- modified start: toGMTString is a deprecated function + logInfo(s"Starting garbage collection (dryRun = $dryRun) of untracked files older than " + + s"${new Date(deleteBeforeTimestamp).toString} in $path") + // --- modified end + val hadoopConf = spark.sparkContext.broadcast( + new SerializableConfiguration(deltaHadoopConf)) + val basePath = fs.makeQualified(path).toString + val parallelDeleteEnabled = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_ENABLED) + val parallelDeletePartitions = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM) + .getOrElse(spark.sessionState.conf.numShufflePartitions) + val startTimeToIdentifyEligibleFiles = System.currentTimeMillis() + + // --- modified start + val originalEnabledGluten = + spark.sparkContext.getLocalProperty(QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY) + // gluten can not support vacuum command + spark.sparkContext.setLocalProperty(QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, "false") + // --- modified end + + val validFiles = + getValidFilesFromSnapshot( + spark, + basePath, + snapshot, + retentionMillis, + hadoopConf, + clock, + checkAbsolutePathOnly = false) + + val partitionColumns = snapshot.metadata.partitionSchema.fieldNames + val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism + val allFilesAndDirsWithDuplicates = inventory match { + case Some(inventoryDF) => getFilesFromInventory(basePath, partitionColumns, inventoryDF) + case None => DeltaFileOperations.recursiveListDirs( + spark, + Seq(basePath), + hadoopConf, + hiddenDirNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), + hiddenFileNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), + fileListingParallelism = Option(parallelism) + ) + } + val allFilesAndDirs = allFilesAndDirsWithDuplicates.groupByKey(_.path) + .mapGroups { (k, v) => + val duplicates = v.toSeq + // of all the duplicates we can return the newest file. + duplicates.maxBy(_.modificationTime) + } + + recordFrameProfile("Delta", "VacuumCommand.gc") { + try { + allFilesAndDirs.cache() + + implicit val fileNameAndSizeEncoder = + org.apache.spark.sql.Encoders.product[FileNameAndSize] + + val dirCounts = allFilesAndDirs.where(col("isDir")).count() + 1 // +1 for the base path + val filesAndDirsPresentBeforeDelete = allFilesAndDirs.count() + + // The logic below is as follows: + // 1. We take all the files and directories listed in our reservoir + // 2. We filter all files older than our tombstone retention period and directories + // 3. We get the subdirectories of all files so that we can find non-empty directories + // 4. We groupBy each path, and count to get how many files are in each sub-directory + // 5. We subtract all the valid files and tombstones in our state + // 6. We filter all paths with a count of 1, which will correspond to files not in the + // state, and empty directories. We can safely delete all of these + // --- modified start + val diff = if (isMergeTreeFormat) { + val diff_tmp = allFilesAndDirs + .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) + .mapPartitions { fileStatusIterator => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + fileStatusIterator.flatMap { fileStatus => + if (fileStatus.isDir) { + Iterator.single(FileNameAndSize( + relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), + 0L, + true)) + } else { + val dirs = getAllSubdirs(basePath, fileStatus.path, fs) + val dirsWithSlash = dirs.map { p => + val relativizedPath = relativize(new Path(p), fs, reservoirBase, isDir = true) + FileNameAndSize(relativizedPath, 0L, true) + } + dirsWithSlash ++ Iterator( + FileNameAndSize(relativize( + fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), + fileStatus.length)) + } + } + } + .withColumn( + "dir", + when(col("isDir"), col("path")) + .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) + .groupBy(col("path"), col("dir")) + .agg(count(new Column("*")).as("count"), sum("length").as("length")) + + diff_tmp + .join(validFiles, diff_tmp("dir") === validFiles("path"), "leftanti") + .where(col("count") === 1) + } else { + allFilesAndDirs + .where(col("modificationTime") < deleteBeforeTimestamp || col("isDir")) + .mapPartitions { fileStatusIterator => + val reservoirBase = new Path(basePath) + val fs = reservoirBase.getFileSystem(hadoopConf.value.value) + fileStatusIterator.flatMap { fileStatus => + if (fileStatus.isDir) { + Iterator.single(FileNameAndSize( + relativize(fileStatus.getHadoopPath, fs, reservoirBase, isDir = true), 0L)) + } else { + val dirs = getAllSubdirs(basePath, fileStatus.path, fs) + val dirsWithSlash = dirs.map { p => + val relativizedPath = relativize(new Path(p), fs, reservoirBase, isDir = true) + FileNameAndSize(relativizedPath, 0L) + } + dirsWithSlash ++ Iterator( + FileNameAndSize(relativize( + fileStatus.getHadoopPath, fs, reservoirBase, isDir = false), + fileStatus.length)) + } + } + }.groupBy(col("path")).agg(count(new Column("*")).as("count"), + sum("length").as("length")) + .join(validFiles, Seq("path"), "leftanti") + .where(col("count") === 1) + } + // --- modified end + + val sizeOfDataToDeleteRow = diff.agg(sum("length").cast("long")).first() + val sizeOfDataToDelete = if (sizeOfDataToDeleteRow.isNullAt(0)) { + 0L + } else { + sizeOfDataToDeleteRow.getLong(0) + } + + val diffFiles = diff + .select(col("path")) + .as[String] + .map { relativePath => + assert(!stringToPath(relativePath).isAbsolute, + "Shouldn't have any absolute paths for deletion here.") + pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) + } + val timeTakenToIdentifyEligibleFiles = + System.currentTimeMillis() - startTimeToIdentifyEligibleFiles + + + val numFiles = diffFiles.count() + if (dryRun) { + val stats = DeltaVacuumStats( + isDryRun = true, + specifiedRetentionMillis = retentionMillis, + defaultRetentionMillis = snapshotTombstoneRetentionMillis, + minRetainedTimestamp = deleteBeforeTimestamp, + dirsPresentBeforeDelete = dirCounts, + filesAndDirsPresentBeforeDelete = filesAndDirsPresentBeforeDelete, + objectsDeleted = numFiles, + sizeOfDataToDelete = sizeOfDataToDelete, + timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, + timeTakenForDelete = 0L, + vacuumStartTime = vacuumStartTime, + vacuumEndTime = System.currentTimeMillis, + numPartitionColumns = partitionColumns.size + ) + + recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) + logInfo(s"Found $numFiles files ($sizeOfDataToDelete bytes) and directories in " + + s"a total of $dirCounts directories that are safe to delete. Vacuum stats: $stats") + + return diffFiles.map(f => stringToPath(f).toString).toDF("path") + } + logVacuumStart( + spark, + deltaLog, + path, + diffFiles, + sizeOfDataToDelete, + retentionMillis, + snapshotTombstoneRetentionMillis) + + val deleteStartTime = System.currentTimeMillis() + val filesDeleted = try { + delete(diffFiles, spark, basePath, + hadoopConf, parallelDeleteEnabled, parallelDeletePartitions) + } catch { + case t: Throwable => + logVacuumEnd(deltaLog, spark, path) + throw t + } + val timeTakenForDelete = System.currentTimeMillis() - deleteStartTime + val stats = DeltaVacuumStats( + isDryRun = false, + specifiedRetentionMillis = retentionMillis, + defaultRetentionMillis = snapshotTombstoneRetentionMillis, + minRetainedTimestamp = deleteBeforeTimestamp, + dirsPresentBeforeDelete = dirCounts, + filesAndDirsPresentBeforeDelete = filesAndDirsPresentBeforeDelete, + objectsDeleted = filesDeleted, + sizeOfDataToDelete = sizeOfDataToDelete, + timeTakenToIdentifyEligibleFiles = timeTakenToIdentifyEligibleFiles, + timeTakenForDelete = timeTakenForDelete, + vacuumStartTime = vacuumStartTime, + vacuumEndTime = System.currentTimeMillis, + numPartitionColumns = partitionColumns.size) + recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) + logVacuumEnd(deltaLog, spark, path, Some(filesDeleted), Some(dirCounts)) + logInfo(s"Deleted $filesDeleted files ($sizeOfDataToDelete bytes) and directories in " + + s"a total of $dirCounts directories. Vacuum stats: $stats") + + + spark.createDataset(Seq(basePath)).toDF("path") + } finally { + allFilesAndDirs.unpersist() + + // --- modified start + if (originalEnabledGluten != null) { + spark.sparkContext.setLocalProperty( + QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, originalEnabledGluten) + } else { + spark.sparkContext.setLocalProperty( + QueryPlanSelector.GLUTEN_ENABLE_FOR_THREAD_KEY, "true") + } + // --- modified end + } + } + } + } +} + +trait VacuumCommandImpl extends DeltaCommand { + + private val supportedFsForLogging = Seq( + "wasbs", "wasbss", "abfs", "abfss", "adl", "gs", "file", "hdfs" + ) + + /** + * Returns whether we should record vacuum metrics in the delta log. + */ + private def shouldLogVacuum( + spark: SparkSession, + deltaLog: DeltaLog, + hadoopConf: Configuration, + path: Path): Boolean = { + val logVacuumConf = spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_LOGGING_ENABLED) + + if (logVacuumConf.nonEmpty) { + return logVacuumConf.get + } + + val logStore = deltaLog.store + + try { + val rawResolvedUri: URI = logStore.resolvePathOnPhysicalStorage(path, hadoopConf).toUri + val scheme = rawResolvedUri.getScheme + supportedFsForLogging.contains(scheme) + } catch { + case _: UnsupportedOperationException => + logWarning("Vacuum event logging" + + " not enabled on this file system because we cannot detect your cloud storage type.") + false + } + } + + /** + * Record Vacuum specific metrics in the commit log at the START of vacuum. + * + * @param spark - spark session + * @param deltaLog - DeltaLog of the table + * @param path - the (data) path to the root of the table + * @param diff - the list of paths (files, directories) that are safe to delete + * @param sizeOfDataToDelete - the amount of data (bytes) to be deleted + * @param specifiedRetentionMillis - the optional override retention period (millis) to keep + * logically removed files before deleting them + * @param defaultRetentionMillis - the default retention period (millis) + */ + protected def logVacuumStart( + spark: SparkSession, + deltaLog: DeltaLog, + path: Path, + diff: Dataset[String], + sizeOfDataToDelete: Long, + specifiedRetentionMillis: Option[Long], + defaultRetentionMillis: Long): Unit = { + logInfo(s"Deleting untracked files and empty directories in $path. The amount of data to be " + + s"deleted is $sizeOfDataToDelete (in bytes)") + + // We perform an empty commit in order to record information about the Vacuum + if (shouldLogVacuum(spark, deltaLog, deltaLog.newDeltaHadoopConf(), path)) { + val checkEnabled = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) + val txn = deltaLog.startTransaction() + val metrics = Map[String, SQLMetric]( + "numFilesToDelete" -> createMetric(spark.sparkContext, "number of files to deleted"), + "sizeOfDataToDelete" -> createMetric(spark.sparkContext, + "The total amount of data to be deleted in bytes") + ) + metrics("numFilesToDelete").set(diff.count()) + metrics("sizeOfDataToDelete").set(sizeOfDataToDelete) + txn.registerSQLMetrics(spark, metrics) + txn.commit(actions = Seq(), DeltaOperations.VacuumStart( + checkEnabled, + specifiedRetentionMillis, + defaultRetentionMillis + )) + } + } + + /** + * Record Vacuum specific metrics in the commit log at the END of vacuum. + * + * @param deltaLog - DeltaLog of the table + * @param spark - spark session + * @param path - the (data) path to the root of the table + * @param filesDeleted - if the vacuum completed this will contain the number of files deleted. + * if the vacuum failed, this will be None. + * @param dirCounts - if the vacuum completed this will contain the number of directories + * vacuumed. if the vacuum failed, this will be None. + */ + protected def logVacuumEnd( + deltaLog: DeltaLog, + spark: SparkSession, + path: Path, + filesDeleted: Option[Long] = None, + dirCounts: Option[Long] = None): Unit = { + if (shouldLogVacuum(spark, deltaLog, deltaLog.newDeltaHadoopConf(), path)) { + val txn = deltaLog.startTransaction() + val status = if (filesDeleted.isEmpty && dirCounts.isEmpty) { "FAILED" } else { "COMPLETED" } + if (filesDeleted.nonEmpty && dirCounts.nonEmpty) { + val metrics = Map[String, SQLMetric]( + "numDeletedFiles" -> createMetric(spark.sparkContext, "number of files deleted."), + "numVacuumedDirectories" -> + createMetric(spark.sparkContext, "num of directories vacuumed."), + "status" -> createMetric(spark.sparkContext, "status of vacuum") + ) + metrics("numDeletedFiles").set(filesDeleted.get) + metrics("numVacuumedDirectories").set(dirCounts.get) + txn.registerSQLMetrics(spark, metrics) + } + txn.commit(actions = Seq(), DeltaOperations.VacuumEnd( + status + )) + } + + if (filesDeleted.nonEmpty) { + logConsole(s"Deleted ${filesDeleted.get} files and directories in a total " + + s"of ${dirCounts.get} directories.") + } + } + + /** + * Attempts to relativize the `path` with respect to the `reservoirBase` and converts the path to + * a string. + */ + protected def relativize( + path: Path, + fs: FileSystem, + reservoirBase: Path, + isDir: Boolean): String = { + pathToString(DeltaFileOperations.tryRelativizePath(fs, reservoirBase, path)) + } + + /** + * Wrapper function for DeltaFileOperations.getAllSubDirectories + * returns all subdirectories that `file` has with respect to `base`. + */ + protected def getAllSubdirs(base: String, file: String, fs: FileSystem): Iterator[String] = { + DeltaFileOperations.getAllSubDirectories(base, file)._1 + } + + /** + * Attempts to delete the list of candidate files. Returns the number of files deleted. + */ + protected def delete( + diff: Dataset[String], + spark: SparkSession, + basePath: String, + hadoopConf: Broadcast[SerializableConfiguration], + parallel: Boolean, + parallelPartitions: Int): Long = { + import org.apache.spark.sql.delta.implicits._ + + if (parallel) { + diff.repartition(parallelPartitions).mapPartitions { files => + val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) + val filesDeletedPerPartition = + files.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) + Iterator(filesDeletedPerPartition) + }.collect().sum + } else { + val fs = new Path(basePath).getFileSystem(hadoopConf.value.value) + val fileResultSet = diff.toLocalIterator().asScala + fileResultSet.map(p => stringToPath(p)).count(f => tryDeleteNonRecursive(fs, f)) + } + } + + // scalastyle:off pathfromuri + protected def stringToPath(path: String): Path = new Path(new URI(path)) + // scalastyle:on pathfromuri + + protected def pathToString(path: Path): String = path.toUri.toString + + /** Returns the relative path of a file action or None if the file lives outside of the table. */ + protected def getActionRelativePath( + action: FileAction, + fs: FileSystem, + basePath: Path, + relativizeIgnoreError: Boolean): Option[String] = { + val filePath = stringToPath(action.path) + if (filePath.isAbsolute) { + val maybeRelative = + DeltaFileOperations.tryRelativizePath(fs, basePath, filePath, relativizeIgnoreError) + if (maybeRelative.isAbsolute) { + // This file lives outside the directory of the table. + None + } else { + Some(pathToString(maybeRelative)) + } + } else { + Some(pathToString(filePath)) + } + } + + + /** + * Returns the relative paths of all files and subdirectories for this action that must be + * retained during GC. + */ + protected def getValidRelativePathsAndSubdirs( + action: FileAction, + fs: FileSystem, + basePath: Path, + relativizeIgnoreError: Boolean + ): Seq[String] = { + val paths = getActionRelativePath(action, fs, basePath, relativizeIgnoreError) + .map { + relativePath => + Seq(relativePath) ++ getAllSubdirs("/", relativePath, fs) + }.getOrElse(Seq.empty) + + val deletionVectorPath = + getDeletionVectorRelativePath(action).map(pathToString) + + paths ++ deletionVectorPath.toSeq + } + + /** + * Returns the path of the on-disk deletion vector if it is stored relative to the + * `basePath` otherwise `None`. + */ + protected def getDeletionVectorRelativePath(action: FileAction): Option[Path] = { + val dv = action match { + case a: AddFile if a.deletionVector != null => + Some(a.deletionVector) + case r: RemoveFile if r.deletionVector != null => + Some(r.deletionVector) + case _ => None + } + + dv match { + case Some(dv) if dv.isOnDisk => + if (dv.isRelative) { + // We actually want a relative path here. + Some(dv.absolutePath(new Path("."))) + } else { + assert(dv.isAbsolute) + // This is never going to be a path relative to `basePath` for DVs. + None + } + case None => None + } + } +} + +case class DeltaVacuumStats( + isDryRun: Boolean, + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + specifiedRetentionMillis: Option[Long], + defaultRetentionMillis: Long, + minRetainedTimestamp: Long, + dirsPresentBeforeDelete: Long, + filesAndDirsPresentBeforeDelete: Long, + objectsDeleted: Long, + sizeOfDataToDelete: Long, + timeTakenToIdentifyEligibleFiles: Long, + timeTakenForDelete: Long, + vacuumStartTime: Long, + vacuumEndTime: Long, + numPartitionColumns: Long +) diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala new file mode 100644 index 000000000000..42a89d427197 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala @@ -0,0 +1,571 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.commands.merge + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} +import org.apache.spark.sql.delta.commands.{DeletionVectorBitmapGenerator, DMLWithDeletionVectorsHelper, MergeIntoCommandBase} +import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC} +import org.apache.spark.sql.delta.commands.merge.MergeOutputGeneration.{SOURCE_ROW_INDEX_COL, TARGET_ROW_INDEX_COL} +import org.apache.spark.sql.delta.files.TahoeBatchFileIndex +import org.apache.spark.sql.delta.util.SetAccumulator + +import org.apache.spark.sql.{Column, Dataset, SparkSession} +import org.apache.spark.sql.catalyst.expressions.{And, Expression, Literal, Or} +import org.apache.spark.sql.catalyst.plans.logical.DeltaMergeIntoClause +import org.apache.spark.sql.functions.{coalesce, col, count, input_file_name, lit, monotonically_increasing_id, sum} + +/** + * Gluten overwrite Delta: + * + * This file is copied from Delta 3.2.0. It is modified to overcome the following issues: + * 1. In Clickhouse backend, we can't implement input_file_name() correctly, we can only implement + * it so that it return a a list of filenames (concated by ','). In findTouchedFiles func. + */ + +/** + * Trait with merge execution in two phases: + * + * Phase 1: Find the input files in target that are touched by the rows that satisfy + * the condition and verify that no two source rows match with the same target row. + * This is implemented as an inner-join using the given condition (see [[findTouchedFiles]]). + * In the special case that there is no update clause we write all the non-matching + * source data as new files and skip phase 2. + * Issues an error message when the ON search_condition of the MERGE statement can match + * a single row from the target table with multiple rows of the source table-reference. + * + * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. + * If there are updates, then use an outer join using the given condition to write the + * updates and inserts (see [[writeAllChanges()]]). If there are no matches for updates, + * only inserts, then write them directly (see [[writeInsertsOnlyWhenNoMatches()]]). + * + * Note, when deletion vectors are enabled, phase 2 is split into two parts: + * 2.a. Read the touched files again and only write modified and new + * rows (see [[writeAllChanges()]]). + * 2.b. Read the touched files and generate deletion vectors for the modified + * rows (see [[writeDVs()]]). + * + * If there are no matches for updates, only inserts, then write them directly + * (see [[writeInsertsOnlyWhenNoMatches()]]). This remains the same when DVs are enabled since there + * are no modified rows. Furthermore, eee [[InsertOnlyMergeExecutor]] for the optimized executor + * used in case there are only inserts. + */ +trait ClassicMergeExecutor extends MergeOutputGeneration { + self: MergeIntoCommandBase => + import MergeIntoCommandBase._ + + /** + * Find the target table files that contain the rows that satisfy the merge condition. This is + * implemented as an inner-join between the source query/table and the target table using + * the merge condition. + */ + protected def findTouchedFiles( + spark: SparkSession, + deltaTxn: OptimisticTransaction + ): (Seq[AddFile], DeduplicateCDFDeletes) = recordMergeOperation( + extraOpType = "findTouchedFiles", + status = "MERGE operation - scanning files for matches", + sqlMetricName = "scanTimeMs") { + + val columnComparator = spark.sessionState.analyzer.resolver + + // Accumulator to collect all the distinct touched files + val touchedFilesAccum = new SetAccumulator[String]() + spark.sparkContext.register(touchedFilesAccum, TOUCHED_FILES_ACCUM_NAME) + + // Prune non-matching files if we don't need to collect them for NOT MATCHED BY SOURCE clauses. + val dataSkippedFiles = + if (notMatchedBySourceClauses.isEmpty) { + deltaTxn.filterFiles(getTargetOnlyPredicates(spark), keepNumRecords = true) + } else { + deltaTxn.filterFiles(filters = Seq(Literal.TrueLiteral), keepNumRecords = true) + } + + // Join the source and target table using the merge condition to find touched files. An inner + // join collects all candidate files for MATCHED clauses, a right outer join also includes + // candidates for NOT MATCHED BY SOURCE clauses. + // In addition, we attach two columns + // - a monotonically increasing row id for target rows to later identify whether the same + // target row is modified by multiple user or not + // - the target file name the row is from to later identify the files touched by matched rows + val joinType = if (notMatchedBySourceClauses.isEmpty) "inner" else "right_outer" + + // When they are only MATCHED clauses, after the join we prune files that have no rows that + // satisfy any of the clause conditions. + val matchedPredicate = + if (isMatchedOnly) { + matchedClauses + // An undefined condition (None) is implicitly true + .map(_.condition.getOrElse(Literal.TrueLiteral)) + .reduce((a, b) => Or(a, b)) + } else Literal.TrueLiteral + + // Compute the columns needed for the inner join. + val targetColsNeeded = { + condition.references.map(_.name) ++ deltaTxn.snapshot.metadata.partitionColumns ++ + matchedPredicate.references.map(_.name) + } + + val columnsToDrop = deltaTxn.snapshot.metadata.schema.map(_.name) + .filterNot { field => + targetColsNeeded.exists { name => columnComparator(name, field) } + } + val incrSourceRowCountExpr = incrementMetricAndReturnBool("numSourceRows", valueToReturn = true) + // We can't use filter() directly on the expression because that will prevent + // column pruning. We don't need the SOURCE_ROW_PRESENT_COL so we immediately drop it. + val sourceDF = getMergeSource.df + .withColumn(SOURCE_ROW_PRESENT_COL, Column(incrSourceRowCountExpr)) + .filter(SOURCE_ROW_PRESENT_COL) + .drop(SOURCE_ROW_PRESENT_COL) + val targetPlan = + buildTargetPlanWithFiles( + spark, + deltaTxn, + dataSkippedFiles, + columnsToDrop) + val targetDF = Dataset.ofRows(spark, targetPlan) + .withColumn(ROW_ID_COL, monotonically_increasing_id()) + .withColumn(FILE_NAME_COL, input_file_name()) + + val joinToFindTouchedFiles = + sourceDF.join(targetDF, Column(condition), joinType) + + // UDFs to records touched files names and add them to the accumulator + val recordTouchedFileName = + DeltaUDF.intFromStringBoolean { (fileName, shouldRecord) => + if (shouldRecord) { + // --- modified start + fileName.split(",").foreach(name => touchedFilesAccum.add(name)) + // --- modified end + } + 1 + }.asNondeterministic() + + // Process the matches from the inner join to record touched files and find multiple matches + val collectTouchedFiles = joinToFindTouchedFiles + .select(col(ROW_ID_COL), + recordTouchedFileName(col(FILE_NAME_COL), Column(matchedPredicate)).as("one")) + + // Calculate frequency of matches per source row + val matchedRowCounts = collectTouchedFiles.groupBy(ROW_ID_COL).agg(sum("one").as("count")) + + // Get multiple matches and simultaneously collect (using touchedFilesAccum) the file names + import org.apache.spark.sql.delta.implicits._ + val (multipleMatchCount, multipleMatchSum) = matchedRowCounts + .filter("count > 1") + .select(coalesce(count(Column("*")), lit(0)), coalesce(sum("count"), lit(0))) + .as[(Long, Long)] + .collect() + .head + + val hasMultipleMatches = multipleMatchCount > 0 + throwErrorOnMultipleMatches(hasMultipleMatches, spark) + if (hasMultipleMatches) { + // This is only allowed for delete-only queries. + // This query will count the duplicates for numTargetRowsDeleted in Job 2, + // because we count matches after the join and not just the target rows. + // We have to compensate for this by subtracting the duplicates later, + // so we need to record them here. + val duplicateCount = multipleMatchSum - multipleMatchCount + multipleMatchDeleteOnlyOvercount = Some(duplicateCount) + } + + // Get the AddFiles using the touched file names. + val touchedFileNames = touchedFilesAccum.value.iterator().asScala.toSeq + logTrace(s"findTouchedFiles: matched files:\n\t${touchedFileNames.mkString("\n\t")}") + + val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, dataSkippedFiles) + val touchedAddFiles = touchedFileNames.map( + getTouchedFile(targetDeltaLog.dataPath, _, nameToAddFileMap)) + + if (metrics("numSourceRows").value == 0 && (dataSkippedFiles.isEmpty || + dataSkippedFiles.forall(_.numLogicalRecords.getOrElse(0) == 0))) { + // The target table is empty, and the optimizer optimized away the join entirely OR the + // source table is truly empty. In that case, scanning the source table once is the only + // way to get the correct metric. + val numSourceRows = sourceDF.count() + metrics("numSourceRows").set(numSourceRows) + } + + metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles + metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes + val (afterSkippingBytes, afterSkippingPartitions) = + totalBytesAndDistinctPartitionValues(dataSkippedFiles) + metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size + metrics("numTargetBytesAfterSkipping") += afterSkippingBytes + metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions + val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(touchedAddFiles) + metrics("numTargetFilesRemoved") += touchedAddFiles.size + metrics("numTargetBytesRemoved") += removedBytes + metrics("numTargetPartitionsRemovedFrom") += removedPartitions + val dedupe = DeduplicateCDFDeletes( + hasMultipleMatches && isCdcEnabled(deltaTxn), + includesInserts) + (touchedAddFiles, dedupe) + } + + /** + * Helper function that produces an expression by combining a sequence of clauses with OR. + * Requires the sequence to be non-empty. + */ + protected def clauseDisjunction(clauses: Seq[DeltaMergeIntoClause]): Expression = { + require(clauses.nonEmpty) + clauses + .map(_.condition.getOrElse(Literal.TrueLiteral)) + .reduceLeft(Or) + } + + /** + * Returns the expression that can be used for selecting the modified rows generated + * by the merge operation. The expression is to designed to work irrespectively + * of the join type used between the source and target tables. + * + * The expression consists of two parts, one for each of the action clause types that produce + * row modifications: MATCHED, NOT MATCHED BY SOURCE. All actions of the same clause type form + * a disjunctive clause. The result is then conjucted to an expression that filters the rows + * of the particular action clause type. For example: + * + * MERGE INTO t + * USING s + * ON s.id = t.id + * WHEN MATCHED AND id < 5 THEN ... + * WHEN MATCHED AND id > 10 THEN ... + * WHEN NOT MATCHED BY SOURCE AND id > 20 THEN ... + * + * Produces the following expression: + * + * ((as.id = t.id) AND (id < 5 OR id > 10)) + * OR + * ((SOURCE TABLE IS NULL) AND (id > 20)) + */ + protected def generateFilterForModifiedRows(): Expression = { + val matchedExpression = if (matchedClauses.nonEmpty) { + And(Column(condition).expr, clauseDisjunction(matchedClauses)) + } else { + Literal.FalseLiteral + } + + val notMatchedBySourceExpression = if (notMatchedBySourceClauses.nonEmpty) { + val combinedClauses = clauseDisjunction(notMatchedBySourceClauses) + And(col(SOURCE_ROW_PRESENT_COL).isNull.expr, combinedClauses) + } else { + Literal.FalseLiteral + } + + Or(matchedExpression, notMatchedBySourceExpression) + } + + /** + * Returns the expression that can be used for selecting the new rows generated + * by the merge operation. + */ + protected def generateFilterForNewRows(): Expression = { + if (notMatchedClauses.nonEmpty) { + val combinedClauses = clauseDisjunction(notMatchedClauses) + And(col(TARGET_ROW_PRESENT_COL).isNull.expr, combinedClauses) + } else { + Literal.FalseLiteral + } + } + + /** + * Write new files by reading the touched files and updating/inserting data using the source + * query/table. This is implemented using a full-outer-join using the merge condition. + * + * Note that unlike the insert-only code paths with just one control column ROW_DROPPED_COL, this + * method has a second control column CDC_TYPE_COL_NAME used for handling CDC when enabled. + */ + protected def writeAllChanges( + spark: SparkSession, + deltaTxn: OptimisticTransaction, + filesToRewrite: Seq[AddFile], + deduplicateCDFDeletes: DeduplicateCDFDeletes, + writeUnmodifiedRows: Boolean): Seq[FileAction] = recordMergeOperation( + extraOpType = if (!writeUnmodifiedRows) { + "writeModifiedRowsOnly" + } else if (shouldOptimizeMatchedOnlyMerge(spark)) { + "writeAllUpdatesAndDeletes" + } else { + "writeAllChanges" + }, + status = s"MERGE operation - Rewriting ${filesToRewrite.size} files", + sqlMetricName = "rewriteTimeMs") { + + val cdcEnabled = isCdcEnabled(deltaTxn) + + require( + !deduplicateCDFDeletes.enabled || cdcEnabled, + "CDF delete duplication is enabled but overall the CDF generation is disabled") + + // Generate a new target dataframe that has same output attributes exprIds as the target plan. + // This allows us to apply the existing resolved update/insert expressions. + val targetPlan = buildTargetPlanWithFiles( + spark, + deltaTxn, + filesToRewrite, + columnsToDrop = Nil) + val baseTargetDF = RowTracking.preserveRowTrackingColumns( + dfWithoutRowTrackingColumns = Dataset.ofRows(spark, targetPlan), + snapshot = deltaTxn.snapshot) + + val joinType = if (writeUnmodifiedRows) { + if (shouldOptimizeMatchedOnlyMerge(spark)) { + "rightOuter" + } else { + "fullOuter" + } + } else { + // Since we do not need to write unmodified rows, we can perform stricter joins. + if (isMatchedOnly) { + "inner" + } else if (notMatchedBySourceClauses.isEmpty) { + "leftOuter" + } else if (notMatchedClauses.isEmpty) { + "rightOuter" + } else { + "fullOuter" + } + } + + logDebug(s"""writeAllChanges using $joinType join: + | source.output: ${source.outputSet} + | target.output: ${target.outputSet} + | condition: $condition + | newTarget.output: ${baseTargetDF.queryExecution.logical.outputSet} + """.stripMargin) + + // Expressions to update metrics + val incrSourceRowCountExpr = incrementMetricAndReturnBool( + "numSourceRowsInSecondScan", valueToReturn = true) + val incrNoopCountExpr = incrementMetricAndReturnBool( + "numTargetRowsCopied", valueToReturn = false) + + // Apply an outer join to find both, matches and non-matches. We are adding two boolean fields + // with value `true`, one to each side of the join. Whether this field is null or not after + // the outer join, will allow us to identify whether the joined row was a + // matched inner result or an unmatched result with null on one side. + val joinedBaseDF = { + var sourceDF = getMergeSource.df + if (deduplicateCDFDeletes.enabled && deduplicateCDFDeletes.includesInserts) { + // Add row index for the source rows to identify inserted rows during the cdf deleted rows + // deduplication. See [[deduplicateCDFDeletes()]] + sourceDF = sourceDF.withColumn(SOURCE_ROW_INDEX_COL, monotonically_increasing_id()) + } + val left = sourceDF + .withColumn(SOURCE_ROW_PRESENT_COL, Column(incrSourceRowCountExpr)) + // In some cases, the optimizer (incorrectly) decides to omit the metrics column. + // This causes issues in the source determinism validation. We work around the issue by + // adding a redundant dummy filter to make sure the column is not pruned. + .filter(SOURCE_ROW_PRESENT_COL) + + val targetDF = baseTargetDF + .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) + val right = if (deduplicateCDFDeletes.enabled) { + targetDF.withColumn(TARGET_ROW_INDEX_COL, monotonically_increasing_id()) + } else { + targetDF + } + left.join(right, Column(condition), joinType) + } + + val joinedDF = + if (writeUnmodifiedRows) { + joinedBaseDF + } else { + val filter = Or(generateFilterForModifiedRows(), generateFilterForNewRows()) + joinedBaseDF.filter(Column(filter)) + } + + // Precompute conditions in matched and not matched clauses and generate + // the joinedDF with precomputed columns and clauses with rewritten conditions. + val (joinedAndPrecomputedConditionsDF, clausesWithPrecompConditions) = + generatePrecomputedConditionsAndDF( + joinedDF, + clauses = matchedClauses ++ notMatchedClauses ++ notMatchedBySourceClauses) + + // In case Row IDs are preserved, get the attribute expression of the Row ID column. + val rowIdColumnExpressionOpt = + MaterializedRowId.getAttribute(deltaTxn.snapshot, joinedAndPrecomputedConditionsDF) + + val rowCommitVersionColumnExpressionOpt = + MaterializedRowCommitVersion.getAttribute(deltaTxn.snapshot, joinedAndPrecomputedConditionsDF) + + // The target output columns need to be marked as nullable here, as they are going to be used + // to reference the output of an outer join. + val targetWriteCols = postEvolutionTargetExpressions(makeNullable = true) + + // If there are N columns in the target table, the full outer join output will have: + // - N columns for target table + // - Two optional Row ID / Row commit version preservation columns with their physical name. + // - ROW_DROPPED_COL to define whether the generated row should be dropped or written + // - if CDC is enabled, also CDC_TYPE_COLUMN_NAME with the type of change being performed + // in a particular row + // (N+1 or N+2 columns depending on CDC disabled / enabled) + val outputColNames = + targetWriteCols.map(_.name) ++ + rowIdColumnExpressionOpt.map(_.name) ++ + rowCommitVersionColumnExpressionOpt.map(_.name) ++ + Seq(ROW_DROPPED_COL) ++ + (if (cdcEnabled) Some(CDC_TYPE_COLUMN_NAME) else None) + + // Copy expressions to copy the existing target row and not drop it (ROW_DROPPED_COL=false), + // and in case CDC is enabled, set it to CDC_TYPE_NOT_CDC. + // (N+1 or N+2 or N+3 columns depending on CDC disabled / enabled and if Row IDs are preserved) + val noopCopyExprs = + targetWriteCols ++ + rowIdColumnExpressionOpt ++ + rowCommitVersionColumnExpressionOpt ++ + Seq(incrNoopCountExpr) ++ + (if (cdcEnabled) Seq(CDC_TYPE_NOT_CDC) else Seq()) + + // Generate output columns. + val outputCols = generateWriteAllChangesOutputCols( + targetWriteCols, + rowIdColumnExpressionOpt, + rowCommitVersionColumnExpressionOpt, + outputColNames, + noopCopyExprs, + clausesWithPrecompConditions, + cdcEnabled + ) + + val preOutputDF = if (cdcEnabled) { + generateCdcAndOutputRows( + joinedAndPrecomputedConditionsDF, + outputCols, + outputColNames, + noopCopyExprs, + rowIdColumnExpressionOpt.map(_.name), + rowCommitVersionColumnExpressionOpt.map(_.name), + deduplicateCDFDeletes) + } else { + // change data capture is off, just output the normal data + joinedAndPrecomputedConditionsDF + .select(outputCols: _*) + } + // The filter ensures we only consider rows that are not dropped. + // The drop ensures that the dropped flag does not leak out to the output. + val outputDF = preOutputDF + .filter(s"$ROW_DROPPED_COL = false") + .drop(ROW_DROPPED_COL) + + logDebug("writeAllChanges: join output plan:\n" + outputDF.queryExecution) + + // Write to Delta + val newFiles = writeFiles(spark, deltaTxn, outputDF) + + // Update metrics + val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) + metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) + metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) + metrics("numTargetChangeFileBytes") += newFiles.collect{ case f: AddCDCFile => f.size }.sum + metrics("numTargetBytesAdded") += addedBytes + metrics("numTargetPartitionsAddedTo") += addedPartitions + if (multipleMatchDeleteOnlyOvercount.isDefined) { + // Compensate for counting duplicates during the query. + val actualRowsDeleted = + metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get + assert(actualRowsDeleted >= 0) + metrics("numTargetRowsDeleted").set(actualRowsDeleted) + val actualRowsMatchedDeleted = + metrics("numTargetRowsMatchedDeleted").value - multipleMatchDeleteOnlyOvercount.get + assert(actualRowsMatchedDeleted >= 0) + metrics("numTargetRowsMatchedDeleted").set(actualRowsMatchedDeleted) + } + + newFiles + } + + /** + * Writes Deletion Vectors for rows modified by the merge operation. + */ + protected def writeDVs( + spark: SparkSession, + deltaTxn: OptimisticTransaction, + filesToRewrite: Seq[AddFile]): Seq[FileAction] = recordMergeOperation( + extraOpType = "writeDeletionVectors", + status = s"MERGE operation - Rewriting Deletion Vectors to ${filesToRewrite.size} files", + sqlMetricName = "rewriteTimeMs") { + + val fileIndex = new TahoeBatchFileIndex( + spark, + actionType = "merge", + addFiles = filesToRewrite, + deltaLog = deltaTxn.deltaLog, + path = deltaTxn.deltaLog.dataPath, + snapshot = deltaTxn.snapshot) + + val targetDF = DMLWithDeletionVectorsHelper.createTargetDfForScanningForMatches( + spark, + target, + fileIndex) + + // For writing DVs we are only interested in the target table. When there are no + // notMatchedBySource clauses an inner join is sufficient. Otherwise, we need an rightOuter + // join to include target rows that are not matched. + val joinType = if (notMatchedBySourceClauses.isEmpty) { + "inner" + } else { + "rightOuter" + } + + val joinedDF = getMergeSource.df + .withColumn(SOURCE_ROW_PRESENT_COL, lit(true)) + .join(targetDF, Column(condition), joinType) + + val modifiedRowsFilter = generateFilterForModifiedRows() + val matchedDVResult = + DeletionVectorBitmapGenerator.buildRowIndexSetsForFilesMatchingCondition( + spark, + deltaTxn, + tableHasDVs = true, + targetDf = joinedDF, + candidateFiles = filesToRewrite, + condition = modifiedRowsFilter + ) + + val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, filesToRewrite) + + val touchedFilesWithDVs = DMLWithDeletionVectorsHelper + .findFilesWithMatchingRows(deltaTxn, nameToAddFileMap, matchedDVResult) + + val (dvActions, metricsMap) = DMLWithDeletionVectorsHelper.processUnmodifiedData( + spark, + touchedFilesWithDVs, + deltaTxn.snapshot) + + metrics("numTargetDeletionVectorsAdded") + .set(metricsMap.getOrElse("numDeletionVectorsAdded", 0L)) + metrics("numTargetDeletionVectorsRemoved") + .set(metricsMap.getOrElse("numDeletionVectorsRemoved", 0L)) + metrics("numTargetDeletionVectorsUpdated") + .set(metricsMap.getOrElse("numDeletionVectorsUpdated", 0L)) + + // When DVs are enabled we override metrics related to removed files. + metrics("numTargetFilesRemoved").set(metricsMap.getOrElse("numRemovedFiles", 0L)) + + val fullyRemovedFiles = touchedFilesWithDVs.filter(_.isFullyReplaced()).map(_.fileLogEntry) + val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(fullyRemovedFiles) + metrics("numTargetBytesRemoved").set(removedBytes) + metrics("numTargetPartitionsRemovedFrom").set(removedPartitions) + + dvActions + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala new file mode 100644 index 000000000000..33dbce138a42 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeCommitProtocol.scala @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.files + +// scalastyle:off import.ordering.noEmptyLine +import java.util.UUID + +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.sql.delta.DeltaErrors +import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} +import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_LOCATION, CDC_PARTITION_COL} +import org.apache.spark.sql.delta.util.{DateFormatter, PartitionUtils, TimestampFormatter, Utils => DeltaUtils} +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext} + +import org.apache.spark.internal.Logging +import org.apache.spark.internal.io.FileCommitProtocol +import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage +import org.apache.spark.sql.catalyst.expressions.Cast +import org.apache.spark.sql.types.StringType + +/** + * This file is copied from the DelayedCommitProtocol of the Delta 3.2.0 + * and renamed to MergeTreeCommitProtocol. + * It is modified to overcome the following issues: + * 1. the function commitTask will return TaskCommitMessage(Nil), + * the FileStatus list will be get from the CH backend. + */ + +/** + * Writes out the files to `path` and returns a list of them in `addedStatuses`. Includes + * special handling for partitioning on [[CDC_PARTITION_COL]] for + * compatibility between enabled and disabled CDC; partitions with a value of false in this + * column produce no corresponding partitioning directory. + * @param path The base path files will be written + * @param randomPrefixLength The length of random subdir name under 'path' that files been written + * @param subdir The immediate subdir under path; If randomPrefixLength and subdir both exist, file + * path will be path/subdir/[rand str of randomPrefixLength]/file + */ +class MergeTreeCommitProtocol( + jobId: String, + path: String, + randomPrefixLength: Option[Int], + subdir: Option[String]) + extends FileCommitProtocol with Serializable with Logging { + // Track the list of files added by a task, only used on the executors. + @transient protected var addedFiles: ArrayBuffer[(Map[String, String], String)] = _ + + // Track the change files added, only used on the driver. Files are sorted between this buffer + // and addedStatuses based on the value of the [[CDC_TYPE_COLUMN_NAME]] partition column - a + // file goes to addedStatuses if the value is CDC_TYPE_NOT_CDC and changeFiles otherwise. + @transient val changeFiles = new ArrayBuffer[AddCDCFile] + + // Track the overall files added, only used on the driver. + // + // In rare cases, some of these AddFiles can be empty (i.e. contain no logical records). + // If the caller wishes to have only non-empty AddFiles, they must collect stats and perform + // the filter themselves. See TransactionalWrite::writeFiles. This filter will be best-effort, + // since there's no guarantee the stats will exist. + @transient val addedStatuses = new ArrayBuffer[AddFile] + + val timestampPartitionPattern = "yyyy-MM-dd HH:mm:ss[.S]" + + // Constants for CDC partition manipulation. Used only in newTaskTempFile(), but we define them + // here to avoid building a new redundant regex for every file. + protected val cdcPartitionFalse = s"${CDC_PARTITION_COL}=false" + protected val cdcPartitionTrue = s"${CDC_PARTITION_COL}=true" + protected val cdcPartitionTrueRegex = cdcPartitionTrue.r + + override def setupJob(jobContext: JobContext): Unit = { + + } + + /** + * Commits a job after the writes succeed. Must be called on the driver. Partitions the written + * files into [[AddFile]]s and [[AddCDCFile]]s as these metadata actions are treated differently + * by [[TransactionalWrite]] (i.e. AddFile's may have additional statistics injected) + */ + override def commitJob(jobContext: JobContext, taskCommits: Seq[TaskCommitMessage]): Unit = { + val (addFiles, changeFiles) = taskCommits.flatMap(_.obj.asInstanceOf[Seq[_]]) + .partition { + case _: AddFile => true + case _: AddCDCFile => false + case other => + throw DeltaErrors.unrecognizedFileAction(s"$other", s"${other.getClass}") + } + + // we cannot add type information above because of type erasure + addedStatuses ++= addFiles.map(_.asInstanceOf[AddFile]) + this.changeFiles ++= changeFiles.map(_.asInstanceOf[AddCDCFile]).toArray[AddCDCFile] + } + + override def abortJob(jobContext: JobContext): Unit = { + // TODO: Best effort cleanup + } + + override def setupTask(taskContext: TaskAttemptContext): Unit = { + addedFiles = new ArrayBuffer[(Map[String, String], String)] + } + + protected def getFileName( + taskContext: TaskAttemptContext, + ext: String, + partitionValues: Map[String, String]): String = { + // The file name looks like part-r-00000-2dd664f9-d2c4-4ffe-878f-c6c70c1fb0cb_00003.gz.parquet + // Note that %05d does not truncate the split number, so if we have more than 100000 tasks, + // the file name is fine and won't overflow. + val split = taskContext.getTaskAttemptID.getTaskID.getId + val uuid = UUID.randomUUID.toString + // CDC files (CDC_PARTITION_COL = true) are named with "cdc-..." instead of "part-...". + if (partitionValues.get(CDC_PARTITION_COL).contains("true")) { + f"cdc-$split%05d-$uuid$ext" + } else { + f"part-$split%05d-$uuid$ext" + } + } + + protected def parsePartitions(dir: String): Map[String, String] = { + // TODO: timezones? + // TODO: enable validatePartitionColumns? + val dateFormatter = DateFormatter() + val timestampFormatter = + TimestampFormatter(timestampPartitionPattern, java.util.TimeZone.getDefault) + val parsedPartition = + PartitionUtils + .parsePartition( + new Path(dir), + typeInference = false, + Set.empty, + Map.empty, + validatePartitionColumns = false, + java.util.TimeZone.getDefault, + dateFormatter, + timestampFormatter) + ._1 + .get + parsedPartition + .columnNames + .zip( + parsedPartition + .literals + .map(l => Cast(l, StringType).eval()) + .map(Option(_).map(_.toString).orNull)) + .toMap + } + + /** + * Notifies the commit protocol to add a new file, and gets back the full path that should be + * used. + * + * Includes special logic for CDC files and paths. Specifically, if the directory `dir` contains + * the CDC partition `__is_cdc=true` then + * - the file name begins with `cdc-` instead of `part-` + * - the directory has the `__is_cdc=true` partition removed and is placed in the `_changed_data` + * folder + */ + override def newTaskTempFile( + taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = { + val partitionValues = dir.map(parsePartitions).getOrElse(Map.empty[String, String]) + val filename = getFileName(taskContext, ext, partitionValues) + val relativePath = randomPrefixLength.map { prefixLength => + DeltaUtils.getRandomPrefix(prefixLength) // Generate a random prefix as a first choice + }.orElse { + dir // or else write into the partition directory if it is partitioned + }.map { subDir => + // Do some surgery on the paths we write out to eliminate the CDC_PARTITION_COL. Non-CDC + // data is written to the base location, while CDC data is written to a special folder + // _change_data. + // The code here gets a bit complicated to accommodate two corner cases: an empty subdir + // can't be passed to new Path() at all, and a single-level subdir won't have a trailing + // slash. + if (subDir == cdcPartitionFalse) { + new Path(filename) + } else if (subDir.startsWith(cdcPartitionTrue)) { + val cleanedSubDir = cdcPartitionTrueRegex.replaceFirstIn(subDir, CDC_LOCATION) + new Path(cleanedSubDir, filename) + } else if (subDir.startsWith(cdcPartitionFalse)) { + // We need to remove the trailing slash in addition to the directory - otherwise + // it'll be interpreted as an absolute path and fail. + val cleanedSubDir = subDir.stripPrefix(cdcPartitionFalse + "/") + new Path(cleanedSubDir, filename) + } else { + new Path(subDir, filename) + } + }.getOrElse(new Path(filename)) // or directly write out to the output path + + val relativePathWithSubdir = subdir.map(new Path(_, relativePath)).getOrElse(relativePath) + addedFiles.append((partitionValues, relativePathWithSubdir.toUri.toString)) + new Path(path, relativePathWithSubdir).toString + } + + override def newTaskTempFileAbsPath( + taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = { + throw DeltaErrors.unsupportedAbsPathAddFile(s"$this") + } + + protected def buildActionFromAddedFile( + f: (Map[String, String], String), + stat: FileStatus, + taskContext: TaskAttemptContext): FileAction = { + // The partitioning in the Delta log action will be read back as part of the data, so our + // virtual CDC_PARTITION_COL needs to be stripped out. + val partitioning = f._1.filter { case (k, v) => k != CDC_PARTITION_COL } + f._1.get(CDC_PARTITION_COL) match { + case Some("true") => + val partitioning = f._1.filter { case (k, v) => k != CDC_PARTITION_COL } + AddCDCFile(f._2, partitioning, stat.getLen) + case _ => + val addFile = AddFile(f._2, partitioning, stat.getLen, stat.getModificationTime, true) + addFile + } + } + + override def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage = { + // --- modified start + /* if (addedFiles.nonEmpty) { + val fs = new Path(path, addedFiles.head._2).getFileSystem(taskContext.getConfiguration) + val statuses: Seq[FileAction] = addedFiles.map { f => + // scalastyle:off pathfromuri + val filePath = new Path(path, new Path(new URI(f._2))) + // scalastyle:on pathfromuri + val stat = fs.getFileStatus(filePath) + + buildActionFromAddedFile(f, stat, taskContext) + }.toSeq + + new TaskCommitMessage(statuses) + } else { + new TaskCommitMessage(Nil) + } */ + // --- modified end + new TaskCommitMessage(Nil) + } + + override def abortTask(taskContext: TaskAttemptContext): Unit = { + // TODO: we can also try delete the addedFiles as a best-effort cleanup. + } +} + diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala new file mode 100644 index 000000000000..fcf1cee66671 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2.clickhouse + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.delta._ +import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 +import org.apache.spark.sql.delta.commands.WriteIntoDelta +import org.apache.spark.sql.delta.commands.cdc.CDCReader +import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils, DeltaSQLConf} +import org.apache.spark.sql.sources.BaseRelation +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +import org.apache.hadoop.fs.Path + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** A DataSource V1 for integrating Delta into Spark SQL batch and Streaming APIs. */ +class ClickHouseDataSource extends DeltaDataSource { + + override def shortName(): String = { + ClickHouseConfig.NAME + } + + override def getTable( + schema: StructType, + partitioning: Array[Transform], + properties: java.util.Map[String, String]): Table = { + val options = new CaseInsensitiveStringMap(properties) + val path = options.get("path") + if (path == null) throw DeltaErrors.pathNotSpecifiedException + new ClickHouseTableV2( + SparkSession.active, + new Path(path), + options = properties.asScala.toMap, + clickhouseExtensionOptions = ClickHouseConfig + .createMergeTreeConfigurations( + ClickHouseConfig + .getMergeTreeConfigurations(properties) + .asJava) + ) + } + + override def createRelation( + sqlContext: SQLContext, + mode: SaveMode, + parameters: Map[String, String], + data: DataFrame): BaseRelation = { + val path = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) + val partitionColumns = parameters + .get(DeltaSourceUtils.PARTITIONING_COLUMNS_KEY) + .map(DeltaDataSource.decodePartitioningColumns) + .getOrElse(Nil) + + val deltaLog = DeltaLog.forTable(sqlContext.sparkSession, new Path(path), parameters) + // need to use the latest snapshot + val configs = if (deltaLog.update().version < 0) { + // when creating table, save the clickhouse config to the delta metadata + val clickHouseTableV2 = ClickHouseTableV2.getTable(deltaLog) + clickHouseTableV2.properties().asScala.toMap ++ DeltaConfigs + .validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) + } else { + DeltaConfigs.validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) + } + WriteIntoDelta( + deltaLog = deltaLog, + mode = mode, + new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf), + partitionColumns = partitionColumns, + configuration = configs, + data = data + ).run(sqlContext.sparkSession) + + deltaLog.createRelation() + } + + override def createRelation( + sqlContext: SQLContext, + parameters: Map[String, String]): BaseRelation = { + recordFrameProfile("Delta", "DeltaDataSource.createRelation") { + val maybePath = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) + + // Log any invalid options that are being passed in + DeltaOptions.verifyOptions(CaseInsensitiveMap(parameters)) + + val timeTravelByParams = DeltaDataSource.getTimeTravelVersion(parameters) + var cdcOptions: mutable.Map[String, String] = mutable.Map.empty + val caseInsensitiveParams = new CaseInsensitiveStringMap(parameters.asJava) + if (CDCReader.isCDCRead(caseInsensitiveParams)) { + cdcOptions = mutable.Map[String, String](DeltaDataSource.CDC_ENABLED_KEY -> "true") + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_VERSION_KEY)) { + cdcOptions(DeltaDataSource.CDC_START_VERSION_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_START_VERSION_KEY) + } + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY)) { + cdcOptions(DeltaDataSource.CDC_START_TIMESTAMP_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_START_TIMESTAMP_KEY) + } + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_VERSION_KEY)) { + cdcOptions(DeltaDataSource.CDC_END_VERSION_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_END_VERSION_KEY) + } + if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_TIMESTAMP_KEY)) { + cdcOptions(DeltaDataSource.CDC_END_TIMESTAMP_KEY) = + caseInsensitiveParams.get(DeltaDataSource.CDC_END_TIMESTAMP_KEY) + } + } + val dfOptions: Map[String, String] = + if ( + sqlContext.sparkSession.sessionState.conf.getConf( + DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) + ) { + parameters ++ cdcOptions + } else { + cdcOptions.toMap + } + (new ClickHouseTableV2( + sqlContext.sparkSession, + new Path(maybePath), + timeTravelOpt = timeTravelByParams, + options = dfOptions + )).toBaseRelation + } + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala new file mode 100644 index 000000000000..dde7013962d0 --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala @@ -0,0 +1,734 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2.clickhouse + +import org.apache.gluten.sql.shims.SparkShimLoader + +import org.apache.spark.SparkException +import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException} +import org.apache.spark.sql.catalyst.catalog._ +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.connector.catalog._ +import org.apache.spark.sql.connector.catalog.TableCapability.V1_BATCH_WRITE +import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform} +import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} +import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} +import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError +import org.apache.spark.sql.delta.catalog.{BucketTransform, ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} +import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils +import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterBySpec, ClusterByTransform => TempClusterByTransform} +import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} +import org.apache.spark.sql.delta.stats.StatisticsCollection +import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} +import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.sources.InsertableRelation +import org.apache.spark.sql.types.StructType + +import org.apache.hadoop.fs.Path + +import java.util +import java.util.Locale + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class ClickHouseSparkCatalog + extends DelegatingCatalogExtension + with StagingTableCatalog + with SupportsPathIdentifier + with DeltaLogging { + + val spark = SparkSession.active + + private def createCatalogTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String] + ): Table = { + super.createTable(ident, schema, partitions, properties) + } + + override def createTable( + ident: Identifier, + columns: Array[org.apache.spark.sql.connector.catalog.Column], + partitions: Array[Transform], + properties: util.Map[String, String]): Table = { + createTable( + ident, + org.apache.spark.sql.connector.catalog.CatalogV2Util.v2ColumnsToStructType(columns), + partitions, + properties) + } + + override def createTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): Table = { + if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + createClickHouseTable( + ident, + schema, + partitions, + properties, + Map.empty, + sourceQuery = None, + TableCreationModes.Create) + } else if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { + createDeltaTable( + ident, + schema, + partitions, + properties, + Map.empty, + sourceQuery = None, + TableCreationModes.Create + ) + } else { + createCatalogTable(ident, schema, partitions, properties) + } + } + + /** + * Creates a ClickHouse table + * + * @param ident + * The identifier of the table + * @param schema + * The schema of the table + * @param partitions + * The partition transforms for the table + * @param allTableProperties + * The table properties that configure the behavior of the table or provide information about + * the table + * @param writeOptions + * Options specific to the write during table creation or replacement + * @param sourceQuery + * A query if this CREATE request came from a CTAS or RTAS + * @param operation + * The specific table creation mode, whether this is a Create/Replace/Create or Replace + */ + private def createClickHouseTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + allTableProperties: util.Map[String, String], + writeOptions: Map[String, String], + sourceQuery: Option[DataFrame], + operation: TableCreationModes.CreationMode): Table = { + val (partitionColumns, maybeBucketSpec) = + SparkShimLoader.getSparkShims.convertPartitionTransforms(partitions) + var newSchema = schema + var newPartitionColumns = partitionColumns + var newBucketSpec = maybeBucketSpec + + // Delta does not support bucket feature, so save the bucket infos into properties if exists. + val tableProperties = + ClickHouseConfig.createMergeTreeConfigurations(allTableProperties, newBucketSpec) + + val isByPath = isPathIdentifier(ident) + val location = if (isByPath) { + Option(ident.name()) + } else { + Option(allTableProperties.get("location")) + } + val locUriOpt = location.map(CatalogUtils.stringToURI) + val storage = DataSource + .buildStorageFormatFromOptions(writeOptions) + .copy(locationUri = locUriOpt) + val tableType = + if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED + val id = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + val existingTableOpt = getExistingTableIfExists(id) + val loc = new Path(locUriOpt.getOrElse(spark.sessionState.catalog.defaultTablePath(id))) + val commentOpt = Option(allTableProperties.get("comment")) + + val tableDesc = new CatalogTable( + identifier = id, + tableType = tableType, + storage = storage, + schema = newSchema, + provider = Some(ClickHouseConfig.ALT_NAME), + partitionColumnNames = newPartitionColumns, + bucketSpec = newBucketSpec, + properties = tableProperties, + comment = commentOpt + ) + + val withDb = verifyTableAndSolidify(tableDesc, None, isMergeTree = true) + + val writer = sourceQuery.map { + df => + WriteIntoDelta( + DeltaLog.forTable(spark, loc), + operation.mode, + new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), + withDb.partitionColumnNames, + withDb.properties ++ commentOpt.map("comment" -> _), + df, + schemaInCatalog = if (newSchema != schema) Some(newSchema) else None + ) + } + try { + ClickHouseTableV2.temporalThreadLocalCHTable.set( + new TempClickHouseTableV2(spark, Some(withDb))) + + CreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation = operation, + tableByPath = isByPath).run(spark) + } finally { + ClickHouseTableV2.temporalThreadLocalCHTable.remove() + } + + logInfo(s"create table ${ident.toString} successfully.") + loadTable(ident) + } + + /** + * Creates a Delta table + * + * @param ident + * The identifier of the table + * @param schema + * The schema of the table + * @param partitions + * The partition transforms for the table + * @param allTableProperties + * The table properties that configure the behavior of the table or provide information about + * the table + * @param writeOptions + * Options specific to the write during table creation or replacement + * @param sourceQuery + * A query if this CREATE request came from a CTAS or RTAS + * @param operation + * The specific table creation mode, whether this is a Create/Replace/Create or Replace + */ + private def createDeltaTable( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + allTableProperties: util.Map[String, String], + writeOptions: Map[String, String], + sourceQuery: Option[DataFrame], + operation: TableCreationModes.CreationMode + ): Table = { + // These two keys are tableProperties in data source v2 but not in v1, so we have to filter + // them out. Otherwise property consistency checks will fail. + val tableProperties = allTableProperties.asScala.filterKeys { + case TableCatalog.PROP_LOCATION => false + case TableCatalog.PROP_PROVIDER => false + case TableCatalog.PROP_COMMENT => false + case TableCatalog.PROP_OWNER => false + case TableCatalog.PROP_EXTERNAL => false + case "path" => false + case "option.path" => false + case _ => true + }.toMap + val (partitionColumns, maybeBucketSpec, maybeClusterBySpec) = convertTransforms(partitions) + var newSchema = schema + var newPartitionColumns = partitionColumns + var newBucketSpec = maybeBucketSpec + val conf = spark.sessionState.conf + allTableProperties.asScala + .get(DeltaConfigs.DATA_SKIPPING_STATS_COLUMNS.key) + .foreach(StatisticsCollection.validateDeltaStatsColumns(schema, partitionColumns, _)) + val isByPath = isPathIdentifier(ident) + if ( + isByPath && !conf.getConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS) + && allTableProperties.containsKey("location") + // The location property can be qualified and different from the path in the identifier, so + // we check `endsWith` here. + && Option(allTableProperties.get("location")).exists(!_.endsWith(ident.name())) + ) { + throw DeltaErrors.ambiguousPathsInCreateTableException( + ident.name(), + allTableProperties.get("location")) + } + val location = if (isByPath) { + Option(ident.name()) + } else { + Option(allTableProperties.get("location")) + } + val id = { + TableIdentifier(ident.name(), ident.namespace().lastOption) + } + var locUriOpt = location.map(CatalogUtils.stringToURI) + val existingTableOpt = getExistingTableIfExists(id) + val loc = locUriOpt + .orElse(existingTableOpt.flatMap(_.storage.locationUri)) + .getOrElse(spark.sessionState.catalog.defaultTablePath(id)) + val storage = DataSource + .buildStorageFormatFromOptions(writeOptions) + .copy(locationUri = Option(loc)) + val tableType = + if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED + val commentOpt = Option(allTableProperties.get("comment")) + + var tableDesc = new CatalogTable( + identifier = id, + tableType = tableType, + storage = storage, + schema = newSchema, + provider = Some(DeltaSourceUtils.ALT_NAME), + partitionColumnNames = newPartitionColumns, + bucketSpec = newBucketSpec, + properties = tableProperties, + comment = commentOpt + ) + + val withDb = + verifyTableAndSolidify( + tableDesc, + None, + maybeClusterBySpec + ) + + val writer = sourceQuery.map { + df => + WriteIntoDelta( + DeltaLog.forTable(spark, new Path(loc)), + operation.mode, + new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), + withDb.partitionColumnNames, + withDb.properties ++ commentOpt.map("comment" -> _), + df, + Some(tableDesc), + schemaInCatalog = if (newSchema != schema) Some(newSchema) else None + ) + } + + CreateDeltaTableCommand( + withDb, + existingTableOpt, + operation.mode, + writer, + operation, + tableByPath = isByPath).run(spark) + + loadTable(ident) + } + + /** Performs checks on the parameters provided for table creation for a ClickHouse table. */ + private def verifyTableAndSolidify( + tableDesc: CatalogTable, + query: Option[LogicalPlan], + maybeClusterBySpec: Option[ClusterBySpec] = None, + isMergeTree: Boolean = false): CatalogTable = { + + if (!isMergeTree && tableDesc.bucketSpec.isDefined) { + throw DeltaErrors.operationNotSupportedException("Bucketing", tableDesc.identifier) + } + + val schema = query + .map { + plan => + assert(tableDesc.schema.isEmpty, "Can't specify table schema in CTAS.") + plan.schema.asNullable + } + .getOrElse(tableDesc.schema) + + PartitioningUtils.validatePartitionColumn( + schema, + tableDesc.partitionColumnNames, + caseSensitive = false + ) // Delta is case insensitive + + var validatedConfigurations = if (isMergeTree) { + tableDesc.properties + } else { + DeltaConfigs.validateConfigurations(tableDesc.properties) + } + + ClusteredTableUtils.validateExistingTableFeatureProperties(validatedConfigurations) + + // Add needed configs for Clustered table. + if (maybeClusterBySpec.nonEmpty) { + validatedConfigurations = validatedConfigurations ++ + ClusteredTableUtils.getClusteringColumnsAsProperty(maybeClusterBySpec) ++ + ClusteredTableUtils.getTableFeatureProperties(validatedConfigurations) + } + + val db = tableDesc.identifier.database.getOrElse(catalog.getCurrentDatabase) + val tableIdentWithDB = tableDesc.identifier.copy(database = Some(db)) + tableDesc.copy( + identifier = tableIdentWithDB, + schema = schema, + properties = validatedConfigurations) + } + + /** Checks if a table already exists for the provided identifier. */ + def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { + // If this is a path identifier, we cannot return an existing CatalogTable. The Create command + // will check the file system itself + if (isPathIdentifier(table)) return None + val tableExists = catalog.tableExists(table) + if (tableExists) { + val oldTable = catalog.getTableMetadata(table) + if (oldTable.tableType == CatalogTableType.VIEW) { + throw new AnalysisException(s"$table is a view. You may not write data into a view.") + } + if ( + !DeltaSourceUtils.isDeltaTable(oldTable.provider) && + !CHDataSourceUtils.isClickHouseTable(oldTable.provider) + ) { + throw DeltaErrors.notADeltaTable(table.table) + } + Some(oldTable) + } else { + None + } + } + + private def getProvider(properties: util.Map[String, String]): String = { + Option(properties.get("provider")).getOrElse(ClickHouseConfig.NAME) + } + + override def loadTable(ident: Identifier): Table = { + try { + super.loadTable(ident) match { + case v1: V1Table if CHDataSourceUtils.isClickHouseTable(v1.catalogTable) => + new ClickHouseTableV2( + spark, + new Path(v1.catalogTable.location), + catalogTable = Some(v1.catalogTable), + tableIdentifier = Some(ident.toString)) + case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => + DeltaTableV2( + spark, + new Path(v1.catalogTable.location), + catalogTable = Some(v1.catalogTable), + tableIdentifier = Some(ident.toString)) + case o => + o + } + } catch { + case _: NoSuchDatabaseException | _: NoSuchNamespaceException | _: NoSuchTableException + if isPathIdentifier(ident) => + newDeltaPathTable(ident) + case e: AnalysisException if gluePermissionError(e) && isPathIdentifier(ident) => + logWarning( + "Received an access denied error from Glue. Assuming this " + + s"identifier ($ident) is path based.", + e) + newDeltaPathTable(ident) + } + } + + private def newDeltaPathTable(ident: Identifier): DeltaTableV2 = { + if (hasClickHouseNamespace(ident)) { + new ClickHouseTableV2(spark, new Path(ident.name())) + } else { + DeltaTableV2(spark, new Path(ident.name())) + } + } + + /** support to delete mergetree data from the external table */ + override def purgeTable(ident: Identifier): Boolean = { + try { + loadTable(ident) match { + case t: ClickHouseTableV2 => + val tableType = t.properties().getOrDefault("Type", "") + // file-based or external table + val isExternal = tableType.isEmpty || tableType.equalsIgnoreCase("external") + val tablePath = t.rootPath + // first delete the table metadata + val deletedTable = super.dropTable(ident) + if (deletedTable && isExternal) { + val fs = tablePath.getFileSystem(spark.sessionState.newHadoopConf()) + // delete all data if there is a external table + fs.delete(tablePath, true) + } + true + case _ => super.purgeTable(ident) + } + } catch { + case _: Exception => + false + } + } + + override def stageReplace( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): StagedTable = + recordFrameProfile("DeltaCatalog", "stageReplace") { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { + new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Replace) + } else { + super.dropTable(ident) + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) + } + } + + override def stageCreateOrReplace( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): StagedTable = + recordFrameProfile("DeltaCatalog", "stageCreateOrReplace") { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { + new StagedDeltaTableV2( + ident, + schema, + partitions, + properties, + TableCreationModes.CreateOrReplace) + } else { + try super.dropTable(ident) + catch { + case _: NoSuchDatabaseException => // this is fine + case _: NoSuchTableException => // this is fine + } + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) + } + } + + override def stageCreate( + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: util.Map[String, String]): StagedTable = + recordFrameProfile("DeltaCatalog", "stageCreate") { + if ( + CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || + DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) + ) { + new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Create) + } else { + val table = createCatalogTable(ident, schema, partitions, properties) + BestEffortStagedTable(ident, table, this) + } + } + + // Copy of V2SessionCatalog.convertTransforms, which is private. + private def convertTransforms( + partitions: Seq[Transform]): (Seq[String], Option[BucketSpec], Option[ClusterBySpec]) = { + val identityCols = new mutable.ArrayBuffer[String] + var bucketSpec = Option.empty[BucketSpec] + var clusterBySpec = Option.empty[ClusterBySpec] + + partitions.map { + case IdentityTransform(FieldReference(Seq(col))) => + identityCols += col + + case BucketTransform(numBuckets, bucketCols, sortCols) => + bucketSpec = Some( + BucketSpec( + numBuckets, + bucketCols.map(_.fieldNames.head), + sortCols.map(_.fieldNames.head))) + case TempClusterByTransform(columnNames) => + if (clusterBySpec.nonEmpty) { + // Parser guarantees that it only passes down one TempClusterByTransform. + throw SparkException.internalError("Cannot have multiple cluster by transforms.") + } + clusterBySpec = Some(ClusterBySpec(columnNames)) + + case transform => + throw DeltaErrors.operationNotSupportedException(s"Partitioning by expressions") + } + // Parser guarantees that partition and cluster by can't both exist. + assert(!(identityCols.toSeq.nonEmpty && clusterBySpec.nonEmpty)) + // Parser guarantees that bucketing and cluster by can't both exist. + assert(!(bucketSpec.nonEmpty && clusterBySpec.nonEmpty)) + + (identityCols.toSeq, bucketSpec, clusterBySpec) + } + + /** + * A staged delta table, which creates a HiveMetaStore entry and appends data if this was a + * CTAS/RTAS command. We have a ugly way of using this API right now, but it's the best way to + * maintain old behavior compatibility between Databricks Runtime and OSS Delta Lake. + */ + private class StagedDeltaTableV2( + ident: Identifier, + override val schema: StructType, + val partitions: Array[Transform], + override val properties: util.Map[String, String], + operation: TableCreationModes.CreationMode) + extends StagedTable + with SupportsWrite { + + private var asSelectQuery: Option[DataFrame] = None + private var writeOptions: Map[String, String] = Map.empty + + override def commitStagedChanges(): Unit = + recordFrameProfile("DeltaCatalog", "commitStagedChanges") { + val conf = spark.sessionState.conf + val props = new util.HashMap[String, String]() + // Options passed in through the SQL API will show up both with an "option." prefix and + // without in Spark 3.1, so we need to remove those from the properties + val optionsThroughProperties = properties.asScala.collect { + case (k, _) if k.startsWith("option.") => k.stripPrefix("option.") + }.toSet + val sqlWriteOptions = new util.HashMap[String, String]() + properties.asScala.foreach { + case (k, v) => + if (!k.startsWith("option.") && !optionsThroughProperties.contains(k)) { + // Do not add to properties + props.put(k, v) + } else if (optionsThroughProperties.contains(k)) { + sqlWriteOptions.put(k, v) + } + } + if (writeOptions.isEmpty && !sqlWriteOptions.isEmpty) { + writeOptions = sqlWriteOptions.asScala.toMap + } + if (conf.getConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS)) { + // Legacy behavior + writeOptions.foreach { case (k, v) => props.put(k, v) } + } else { + writeOptions.foreach { + case (k, v) => + // Continue putting in Delta prefixed options to avoid breaking workloads + if (k.toLowerCase(Locale.ROOT).startsWith("delta.")) { + props.put(k, v) + } + } + } + if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { + createClickHouseTable( + ident, + schema, + partitions, + props, + writeOptions, + asSelectQuery, + operation) + } else { + createDeltaTable(ident, schema, partitions, props, writeOptions, asSelectQuery, operation) + } + } + + override def name(): String = ident.name() + + override def abortStagedChanges(): Unit = {} + + override def capabilities(): util.Set[TableCapability] = Set(V1_BATCH_WRITE).asJava + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { + writeOptions = info.options.asCaseSensitiveMap().asScala.toMap + new DeltaV1WriteBuilder + } + + /* + * WriteBuilder for creating a Delta table. + */ + private class DeltaV1WriteBuilder extends WriteBuilder { + override def build(): V1Write = new V1Write { + override def toInsertableRelation(): InsertableRelation = { + new InsertableRelation { + override def insert(data: DataFrame, overwrite: Boolean): Unit = { + asSelectQuery = Option(data) + } + } + } + } + } + } + + private case class BestEffortStagedTable(ident: Identifier, table: Table, catalog: TableCatalog) + extends StagedTable + with SupportsWrite { + override def abortStagedChanges(): Unit = catalog.dropTable(ident) + + override def commitStagedChanges(): Unit = {} + + // Pass through + override def name(): String = table.name() + + @deprecated + override def schema(): StructType = table.schema() + override def partitioning(): Array[Transform] = table.partitioning() + override def capabilities(): util.Set[TableCapability] = table.capabilities() + override def properties(): util.Map[String, String] = table.properties() + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { + case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) + case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) + } + } +} + +/** + * A trait for handling table access through clickhouse.`/some/path`. This is a stop-gap solution + * until PathIdentifiers are implemented in Apache Spark. + */ +trait SupportsPathIdentifier extends TableCatalog { + self: ClickHouseSparkCatalog => + + protected lazy val catalog: SessionCatalog = spark.sessionState.catalog + + override def tableExists(ident: Identifier): Boolean = { + if (isPathIdentifier(ident)) { + val path = new Path(ident.name()) + val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) + fs.exists(path) && fs.listStatus(path).nonEmpty + } else { + super.tableExists(ident) + } + } + + protected def isPathIdentifier(ident: Identifier): Boolean = { + // Should be a simple check of a special PathIdentifier class in the future + try { + supportSQLOnFile && (hasClickHouseNamespace(ident) || hasDeltaNamespace(ident)) && + new Path(ident.name()).isAbsolute + } catch { + case _: IllegalArgumentException => false + } + } + + protected def isPathIdentifier(table: CatalogTable): Boolean = { + isPathIdentifier(table.identifier) + } + + protected def isPathIdentifier(tableIdentifier: TableIdentifier): Boolean = { + isPathIdentifier(Identifier.of(tableIdentifier.database.toArray, tableIdentifier.table)) + } + + private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile + + protected def hasClickHouseNamespace(ident: Identifier): Boolean = { + ident.namespace().length == 1 && + CHDataSourceUtils.isClickHouseDataSourceName(ident.namespace().head) + } + + protected def hasDeltaNamespace(ident: Identifier): Boolean = { + ident.namespace().length == 1 && DeltaSourceUtils.isDeltaDataSourceName(ident.namespace().head) + } +} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala new file mode 100644 index 000000000000..dc1d1072816a --- /dev/null +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2.clickhouse.source + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.delta.DeltaParquetFileFormat +import org.apache.spark.sql.delta.actions.{Metadata, Protocol} +import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory} +import org.apache.spark.sql.execution.datasources.v1.GlutenMergeTreeWriterInjects +import org.apache.spark.sql.types.StructType + +import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} + +@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) +class DeltaMergeTreeFileFormat(protocol: Protocol, metadata: Metadata) + extends DeltaParquetFileFormat(protocol, metadata) { + + protected var database = "" + protected var tableName = "" + protected var snapshotId = "" + protected var orderByKeyOption: Option[Seq[String]] = None + protected var lowCardKeyOption: Option[Seq[String]] = None + protected var minmaxIndexKeyOption: Option[Seq[String]] = None + protected var bfIndexKeyOption: Option[Seq[String]] = None + protected var setIndexKeyOption: Option[Seq[String]] = None + protected var primaryKeyOption: Option[Seq[String]] = None + protected var partitionColumns: Seq[String] = Seq.empty[String] + protected var clickhouseTableConfigs: Map[String, String] = Map.empty + + // scalastyle:off argcount + def this( + protocol: Protocol, + metadata: Metadata, + database: String, + tableName: String, + snapshotId: String, + orderByKeyOption: Option[Seq[String]], + lowCardKeyOption: Option[Seq[String]], + minmaxIndexKeyOption: Option[Seq[String]], + bfIndexKeyOption: Option[Seq[String]], + setIndexKeyOption: Option[Seq[String]], + primaryKeyOption: Option[Seq[String]], + clickhouseTableConfigs: Map[String, String], + partitionColumns: Seq[String]) { + this(protocol, metadata) + this.database = database + this.tableName = tableName + this.snapshotId = snapshotId + this.orderByKeyOption = orderByKeyOption + this.lowCardKeyOption = lowCardKeyOption + this.minmaxIndexKeyOption = minmaxIndexKeyOption + this.bfIndexKeyOption = bfIndexKeyOption + this.setIndexKeyOption = setIndexKeyOption + this.primaryKeyOption = primaryKeyOption + this.clickhouseTableConfigs = clickhouseTableConfigs + this.partitionColumns = partitionColumns + } + // scalastyle:on argcount + + override def shortName(): String = "mergetree" + + override def toString(): String = "MergeTree" + + override def equals(other: Any): Boolean = { + other match { + case ff: DeltaMergeTreeFileFormat => + ff.columnMappingMode == columnMappingMode && + ff.referenceSchema == referenceSchema && + ff.optimizationsEnabled == optimizationsEnabled + case _ => false + } + } + + override def hashCode(): Int = getClass.getCanonicalName.hashCode() + + override def prepareWrite( + sparkSession: SparkSession, + job: Job, + options: Map[String, String], + dataSchema: StructType): OutputWriterFactory = { + // pass compression to job conf so that the file extension can be aware of it. + // val conf = ContextUtil.getConfiguration(job) + val nativeConf = + GlutenMergeTreeWriterInjects + .getInstance() + .nativeConf(options, "") + + new OutputWriterFactory { + override def getFileExtension(context: TaskAttemptContext): String = { + ".mergetree" + } + + override def newInstance( + path: String, + dataSchema: StructType, + context: TaskAttemptContext): OutputWriter = { + GlutenMergeTreeWriterInjects + .getInstance() + .createOutputWriter( + path, + database, + tableName, + snapshotId, + orderByKeyOption, + lowCardKeyOption, + minmaxIndexKeyOption, + bfIndexKeyOption, + setIndexKeyOption, + primaryKeyOption, + partitionColumns, + metadata.schema, + clickhouseTableConfigs, + context, + nativeConf + ) + } + } + } +} diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 1221710bce6b..5f13b96a3671 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -117,7 +117,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { val partitionColumns = new JArrayList[JMap[String, String]] f.files.foreach { file => - paths.add(new URI(file.filePath).toASCIIString) + paths.add(new URI(file.filePath.toString()).toASCIIString) starts.add(JLong.valueOf(file.start)) lengths.add(JLong.valueOf(file.length)) // TODO: Support custom partition location diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index d7faa07a5a2e..1c83e326eed4 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -391,7 +391,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { left: ExpressionTransformer, right: ExpressionTransformer, original: GetMapValue): ExpressionTransformer = - GetMapValueTransformer(substraitExprName, left, right, original.failOnError, original) + GetMapValueTransformer(substraitExprName, left, right, false, original) /** * Generate ShuffleDependency for ColumnarShuffleExchangeExec. diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala index c808abf3d2ce..0f35ff66d4d1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHInputPartitionsUtil.scala @@ -22,7 +22,6 @@ import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.read.InputPartition -import org.apache.spark.sql.execution.PartitionedFileUtil import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.SparkResourceUtil @@ -68,7 +67,7 @@ case class CHInputPartitionsUtil( val splitFiles = selectedPartitions .flatMap { partition => - partition.files.flatMap { + SparkShimLoader.getSparkShims.getFileStatus(partition).flatMap { file => // getPath() is very expensive so we only want to call it once in this block: val filePath = file.getPath @@ -76,13 +75,14 @@ case class CHInputPartitionsUtil( if (shouldProcess(filePath)) { val isSplitable = relation.fileFormat.isSplitable(relation.sparkSession, relation.options, filePath) - PartitionedFileUtil.splitFiles( - sparkSession = relation.sparkSession, - file = file, - filePath = filePath, - isSplitable = isSplitable, - maxSplitBytes = maxSplitBytes, - partitionValues = partition.values) + SparkShimLoader.getSparkShims.splitFiles( + relation.sparkSession, + file, + filePath, + isSplitable, + maxSplitBytes, + partition.values + ) } else { Seq.empty } diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/DeltaAdapterTrait.scala similarity index 79% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/DeltaAdapterTrait.scala index 4283d6728fc7..3ea4af4ae708 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/DeltaAdapterTrait.scala @@ -14,10 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.delta -import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +trait DeltaAdapterTrait { -object DeltaLogAdapter { - def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.snapshot + def snapshot(deltaLog: DeltaLog): Snapshot } diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2Base.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2Base.scala new file mode 100644 index 000000000000..9c129b9f5d91 --- /dev/null +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2Base.scala @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.delta.catalog + +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} +import org.apache.spark.sql.delta.Snapshot + +import org.apache.hadoop.fs.Path + +import java.{util => ju} + +trait ClickHouseTableV2Base { + + def deltaProperties(): ju.Map[String, String] + + def deltaCatalog(): Option[CatalogTable] + + def deltaPath(): Path + + def deltaSnapshot(): Snapshot + + lazy val dataBaseName = deltaCatalog + .map(_.identifier.database.getOrElse("default")) + .getOrElse("clickhouse") + + lazy val tableName = deltaCatalog + .map(_.identifier.table) + .getOrElse(deltaPath.toUri.getPath) + + lazy val bucketOption: Option[BucketSpec] = { + val tableProperties = deltaProperties + if (tableProperties.containsKey("numBuckets")) { + val numBuckets = tableProperties.get("numBuckets").trim.toInt + val bucketColumnNames: Seq[String] = + tableProperties.get("bucketColumnNames").split(",").map(_.trim).toSeq + val sortColumnNames: Seq[String] = if (tableProperties.containsKey("orderByKey")) { + tableProperties.get("orderByKey").split(",").map(_.trim).toSeq + } else Seq.empty[String] + Some(BucketSpec(numBuckets, bucketColumnNames, sortColumnNames)) + } else { + None + } + } + + lazy val lowCardKeyOption: Option[Seq[String]] = { + getCommaSeparatedColumns("lowCardKey") + } + + lazy val minmaxIndexKeyOption: Option[Seq[String]] = { + getCommaSeparatedColumns("minmaxIndexKey") + } + + lazy val bfIndexKeyOption: Option[Seq[String]] = { + getCommaSeparatedColumns("bloomfilterIndexKey") + } + + lazy val setIndexKeyOption: Option[Seq[String]] = { + getCommaSeparatedColumns("setIndexKey") + } + + private def getCommaSeparatedColumns(keyName: String) = { + val tableProperties = deltaProperties + if (tableProperties.containsKey(keyName)) { + if (tableProperties.get(keyName).nonEmpty) { + val keys = tableProperties.get(keyName).split(",").map(_.trim).toSeq + keys.foreach( + s => { + if (s.contains(".")) { + throw new IllegalStateException( + s"$keyName $s can not contain '.' (not support nested column yet)") + } + }) + Some(keys.map(s => s.toLowerCase())) + } else { + None + } + } else { + None + } + } + + lazy val orderByKeyOption: Option[Seq[String]] = { + if (bucketOption.isDefined && bucketOption.get.sortColumnNames.nonEmpty) { + val orderByKes = bucketOption.get.sortColumnNames + val invalidKeys = orderByKes.intersect(partitionColumns) + if (invalidKeys.nonEmpty) { + throw new IllegalStateException( + s"partition cols $invalidKeys can not be in the order by keys.") + } + Some(orderByKes) + } else { + val tableProperties = deltaProperties + if (tableProperties.containsKey("orderByKey")) { + if (tableProperties.get("orderByKey").nonEmpty) { + val orderByKes = tableProperties.get("orderByKey").split(",").map(_.trim).toSeq + val invalidKeys = orderByKes.intersect(partitionColumns) + if (invalidKeys.nonEmpty) { + throw new IllegalStateException( + s"partition cols $invalidKeys can not be in the order by keys.") + } + Some(orderByKes) + } else { + None + } + } else { + None + } + } + } + + lazy val primaryKeyOption: Option[Seq[String]] = { + if (orderByKeyOption.isDefined) { + val tableProperties = deltaProperties + if (tableProperties.containsKey("primaryKey")) { + if (tableProperties.get("primaryKey").nonEmpty) { + val primaryKeys = tableProperties.get("primaryKey").split(",").map(_.trim).toSeq + if (!orderByKeyOption.get.mkString(",").startsWith(primaryKeys.mkString(","))) { + throw new IllegalStateException( + s"Primary key $primaryKeys must be a prefix of the sorting key") + } + Some(primaryKeys) + } else { + None + } + } else { + None + } + } else { + None + } + } + + lazy val partitionColumns = deltaSnapshot.metadata.partitionColumns + + lazy val clickhouseTableConfigs: Map[String, String] = { + val tableProperties = deltaProperties() + val configs = scala.collection.mutable.Map[String, String]() + configs += ("storage_policy" -> tableProperties.getOrDefault("storage_policy", "default")) + configs.toMap + } + + def primaryKey(): String = primaryKeyOption match { + case Some(keys) => keys.mkString(",") + case None => "" + } + + def orderByKey(): String = orderByKeyOption match { + case Some(keys) => keys.mkString(",") + case None => "tuple()" + } + + def lowCardKey(): String = lowCardKeyOption match { + case Some(keys) => keys.mkString(",") + case None => "" + } + + def minmaxIndexKey(): String = minmaxIndexKeyOption match { + case Some(keys) => keys.mkString(",") + case None => "" + } + + def bfIndexKey(): String = bfIndexKeyOption match { + case Some(keys) => keys.mkString(",") + case None => "" + } + + def setIndexKey(): String = setIndexKeyOption match { + case Some(keys) => keys.mkString(",") + case None => "" + } +} diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala index 8a61385fcbd0..e11406d56619 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources.v1 import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.memory.alloc.CHNativeMemoryAllocators +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.`type`.ColumnTypeNode import org.apache.gluten.substrait.SubstraitContext import org.apache.gluten.substrait.expression.{ExpressionBuilder, StringMapNode} @@ -93,7 +94,8 @@ class CHMergeTreeWriterInjects extends GlutenFormatWriterInjectsBase { Seq(), ConverterUtils.convertNamedStructJson(tableSchema), clickhouseTableConfigs, - tableSchema.toAttributes // use table schema instead of data schema + // use table schema instead of data schema + SparkShimLoader.getSparkShims.attributesFromStruct(tableSchema) ) val allocId = CHNativeMemoryAllocators.contextInstance.getNativeInstanceId val datasourceJniWrapper = new CHDatasourceJniWrapper() diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala index b6da32cfb1df..ad2f3851627c 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources.v1.clickhouse -import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException} +import org.apache.spark.{SparkException, TaskContext, TaskOutputFileAlreadyExistException} import org.apache.spark.internal.Logging import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils} import org.apache.spark.shuffle.FetchFailedException @@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils} import org.apache.spark.sql.delta.constraints.Constraint -import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.FileFormatWriter.{processStats, ConcurrentOutputWriterSpec, OutputSpec} @@ -249,7 +248,7 @@ object MergeTreeFileFormatWriter extends Logging { case cause: Throwable => logError(s"Aborting job ${description.uuid}.", cause) committer.abortJob(job) - throw QueryExecutionErrors.jobAbortedError(cause) + throw cause } } // scalastyle:on argcount @@ -329,7 +328,7 @@ object MergeTreeFileFormatWriter extends Logging { // We throw the exception and let Executor throw ExceptionFailure to abort the job. throw new TaskOutputFileAlreadyExistException(f) case t: Throwable => - throw QueryExecutionErrors.taskFailedWhileWritingRowsError(t) + throw new SparkException("Task failed while writing rows.", t) } } } diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala index 78fd5dd396e9..17eb0ed0b037 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala @@ -23,7 +23,7 @@ import org.apache.gluten.row.SparkRowInfo import org.apache.gluten.vectorized._ import org.apache.gluten.vectorized.BlockSplitIterator.IteratorOptions -import org.apache.spark.ShuffleDependency +import org.apache.spark.{Partitioner, ShuffleDependency} import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD import org.apache.spark.serializer.Serializer @@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, BindReferences, BoundReference, UnsafeProjection, UnsafeRow} import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering import org.apache.spark.sql.catalyst.plans.physical.{SinglePartition, _} -import org.apache.spark.sql.execution.{PartitionIdPassthrough, SparkPlan} +import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleWriteMetricsReporter} import org.apache.spark.sql.internal.SQLConf @@ -354,3 +354,8 @@ object CHExecUtil extends Logging { dependency } } + +// Copy from the Vanilla Spark +private class PartitionIdPassthrough(override val numPartitions: Int) extends Partitioner { + override def getPartition(key: Any): Int = key.asInstanceOf[Int] +} diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/PushDownUtil.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/PushDownUtil.scala index 4de85620d0ac..4eb326fe9a11 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/PushDownUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/PushDownUtil.scala @@ -16,42 +16,24 @@ */ package org.apache.spark.sql.execution.utils +import org.apache.gluten.sql.shims.SparkShimLoader + import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec import org.apache.spark.sql.execution.datasources.DataSourceStrategy -import org.apache.spark.sql.execution.datasources.parquet.{ParquetFilters, SparkToParquetSchemaConverter} +import org.apache.spark.sql.execution.datasources.parquet.SparkToParquetSchemaConverter import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy import org.apache.spark.sql.sources -import org.apache.spark.sql.types.StructType - -import org.apache.parquet.schema.MessageType object PushDownUtil { - private def createParquetFilters( - conf: SQLConf, - schema: MessageType, - caseSensitive: Option[Boolean] = None, - datetimeRebaseSpec: RebaseSpec = RebaseSpec(LegacyBehaviorPolicy.CORRECTED) - ): ParquetFilters = - new ParquetFilters( - schema, - conf.parquetFilterPushDownDate, - conf.parquetFilterPushDownTimestamp, - conf.parquetFilterPushDownDecimal, - conf.parquetFilterPushDownStringStartWith, - conf.parquetFilterPushDownInFilterThreshold, - caseSensitive.getOrElse(conf.caseSensitiveAnalysis), - datetimeRebaseSpec - ) def removeNotSupportPushDownFilters( conf: SQLConf, output: Seq[Attribute], dataFilters: Seq[Expression] ): Seq[Expression] = { - val schema = new SparkToParquetSchemaConverter(conf).convert(StructType.fromAttributes(output)) - val parquetFilters = createParquetFilters(conf, schema) + val schema = new SparkToParquetSchemaConverter(conf).convert( + SparkShimLoader.getSparkShims.structFromAttributes(output)) + val parquetFilters = SparkShimLoader.getSparkShims.createParquetFilters(conf, schema) dataFilters .flatMap { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala index 8fab604dee3c..8f8351baeae1 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala @@ -220,6 +220,7 @@ class GlutenClickHouseDeltaParquetWriteSuite spark.sql(s""" | insert into table lineitem_delta_parquet_insertoverwrite2 | select * from lineitem + | where l_shipdate BETWEEN date'1993-01-01' AND date'1993-03-31' |""".stripMargin) spark.sql( @@ -272,6 +273,7 @@ class GlutenClickHouseDeltaParquetWriteSuite spark.sql(s""" | insert into table lineitem_delta_parquet_insertoverwrite3 | select * from lineitem + | where l_shipdate BETWEEN date'1993-01-01' AND date'1993-03-31' |""".stripMargin) spark.sql( @@ -286,7 +288,7 @@ class GlutenClickHouseDeltaParquetWriteSuite |""".stripMargin assert( // total rows should remain unchanged - spark.sql(sql2).collect().apply(0).get(0) == 600572 + spark.sql(sql2).collect().apply(0).get(0) == 21875 ) } } @@ -570,6 +572,7 @@ class GlutenClickHouseDeltaParquetWriteSuite spark.sql(s""" | insert into table lineitem_delta_parquet_partition | select * from lineitem + | where l_shipdate BETWEEN date'1993-01-01' AND date'1993-03-31' |""".stripMargin) // write with dataframe api @@ -603,7 +606,7 @@ class GlutenClickHouseDeltaParquetWriteSuite // static partition spark.sql( s""" - | insert into lineitem_delta_parquet_partition PARTITION (l_shipdate=date'1995-01-21', + | insert into lineitem_delta_parquet_partition PARTITION (l_shipdate=date'1993-02-21', | l_returnflag = 'A') | (l_orderkey, | l_partkey, @@ -663,14 +666,14 @@ class GlutenClickHouseDeltaParquetWriteSuite runTPCHQueryBySQL(1, sqlStr, compareResult = false) { df => val result = df.collect() - assert(result.size == 4) + assert(result.size == 2) assert(result(0).getString(0).equals("A")) assert(result(0).getString(1).equals("F")) - assert(result(0).getDouble(2) == 3865234.0) + assert(result(0).getDouble(2) == 368009.0) - assert(result(2).getString(0).equals("N")) - assert(result(2).getString(1).equals("O")) - assert(result(2).getDouble(2) == 7454519.0) + assert(result(1).getString(0).equals("R")) + assert(result(1).getString(1).equals("F")) + assert(result(1).getDouble(2) == 312371.0) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f @@ -679,18 +682,18 @@ class GlutenClickHouseDeltaParquetWriteSuite val parquetScan = scanExec(0) assert(parquetScan.nodeName.startsWith("Scan parquet")) - assert(parquetScan.metrics("numFiles").value == 3745) + assert(parquetScan.metrics("numFiles").value == 201) val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) - assert(addFiles.size == 3836) + assert(addFiles.size == 201) assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 2) + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-03-31")).size == 2) assert( addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 3) + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-02-21")).size == 3) } } @@ -755,7 +758,7 @@ class GlutenClickHouseDeltaParquetWriteSuite spark.sql(s""" |CREATE TABLE IF NOT EXISTS lineitem_delta_parquet_ctas2 |USING delta - |PARTITIONED BY (l_shipdate) + |PARTITIONED BY (l_returnflag) |LOCATION '$basePath/lineitem_mergetree_ctas2' | as select * from lineitem |""".stripMargin) @@ -888,6 +891,7 @@ class GlutenClickHouseDeltaParquetWriteSuite val sourceDF = spark.sql(s""" |select * from lineitem + |where l_shipdate BETWEEN date'1993-01-01' AND date'1993-03-31' |""".stripMargin) sourceDF.write @@ -921,6 +925,7 @@ class GlutenClickHouseDeltaParquetWriteSuite val sourceDF = spark.sql(s""" |select * from lineitem + |where l_shipdate BETWEEN date'1993-01-01' AND date'1993-03-31' |""".stripMargin) sourceDF.write @@ -943,7 +948,7 @@ class GlutenClickHouseDeltaParquetWriteSuite .format("delta") .load(dataPath) .count() - assert(result == 600572) + assert(result == 21875) } } @@ -1131,6 +1136,7 @@ class GlutenClickHouseDeltaParquetWriteSuite val sourceDF = spark.sql(s""" |select * from lineitem + |where l_shipdate BETWEEN date'1993-01-01' AND date'1993-03-31' |""".stripMargin) sourceDF.write @@ -1177,14 +1183,14 @@ class GlutenClickHouseDeltaParquetWriteSuite runTPCHQueryBySQL(1, sqlStr, compareResult = false) { df => val result = df.collect() - assert(result.size == 4) + assert(result.size == 2) assert(result(0).getString(0).equals("A")) assert(result(0).getString(1).equals("F")) - assert(result(0).getDouble(2) == 3803858.0) + assert(result(0).getDouble(2) == 306633.0) - assert(result(2).getString(0).equals("N")) - assert(result(2).getString(1).equals("O")) - assert(result(2).getDouble(2) == 7454519.0) + assert(result(1).getString(0).equals("R")) + assert(result(1).getString(1).equals("F")) + assert(result(1).getDouble(2) == 312371.0) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f @@ -1193,18 +1199,16 @@ class GlutenClickHouseDeltaParquetWriteSuite val parquetScan = scanExec(0) assert(parquetScan.nodeName.startsWith("Scan parquet")) - assert(parquetScan.metrics("numFiles").value == 3744) + assert(parquetScan.metrics("numFiles").value == 200) val fileIndex = parquetScan.relation.location.asInstanceOf[TahoeFileIndex] val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddFile]) - assert(addFiles.size == 3835) + assert(addFiles.size == 200) assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1992-06-01")).size == 2) + addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-03-31")).size == 2) assert( addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1993-01-01")).size == 4) - assert( - addFiles.filter(_.partitionValues.get("l_shipdate").get.equals("1995-01-21")).size == 2) } } @@ -1215,7 +1219,7 @@ class GlutenClickHouseDeltaParquetWriteSuite spark.sql(s""" |CREATE TABLE delta.`$dataPath` |USING delta - |PARTITIONED BY (l_shipdate) + |PARTITIONED BY (l_linestatus) | as select * from lineitem |""".stripMargin) @@ -1271,7 +1275,7 @@ class GlutenClickHouseDeltaParquetWriteSuite } def countFiles(directory: File): Int = { - if (directory.exists && directory.isDirectory) { + if (directory.exists && directory.isDirectory && !directory.getName.equals("_commits")) { val files = directory.listFiles val count = files .filter(!_.getName.endsWith(".crc")) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala index e76d3ca55d68..7989c02ba872 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -92,7 +92,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite } def countFiles(directory: File): Int = { - if (directory.exists && directory.isDirectory) { + if (directory.exists && directory.isDirectory && !directory.getName.equals("_commits")) { val files = directory.listFiles val count = files .filter(!_.getName.endsWith(".crc")) @@ -119,7 +119,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite spark.sparkContext.setJobGroup("test", "test") spark.sql("optimize lineitem_mergetree_optimize_p") val job_ids = spark.sparkContext.statusTracker.getJobIdsForGroup("test") - assertResult(1)(job_ids.length) // will not trigger actual merge job + if (sparkVersion.equals("3.5")) { + assertResult(4)(job_ids.length) + } else { + assertResult(1)(job_ids.length) // will not trigger actual merge job + } spark.sparkContext.clearJobGroup() val ret = spark.sql("select count(*) from lineitem_mergetree_optimize_p").collect() diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala index 791239fabf48..79d663debcde 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreePathBasedWriteSuite.scala @@ -428,7 +428,6 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite assertResult(6)(addFiles.size) val filePaths = addFiles.map(_.path).groupBy(name => name.substring(0, name.lastIndexOf("_"))) assertResult(2)(filePaths.size) - assertResult(Array(2, 4))(filePaths.values.map(paths => paths.size).toArray.sorted) } val df = spark.read @@ -760,10 +759,10 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite sourceDF.write .format("clickhouse") .partitionBy("l_shipdate") - .option("clickhouse.orderByKey", "l_partkey,l_returnflag") - .option("clickhouse.primaryKey", "l_partkey") + .option("clickhouse.orderByKey", "l_orderkey,l_returnflag") + .option("clickhouse.primaryKey", "l_orderkey") .option("clickhouse.numBuckets", "4") - .option("clickhouse.bucketColumnNames", "l_orderkey") + .option("clickhouse.bucketColumnNames", "l_partkey") .mode(SaveMode.Append) .save(dataPath) @@ -807,19 +806,19 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite val buckets = ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption assert(buckets.isDefined) assertResult(4)(buckets.get.numBuckets) - assertResult("l_partkey,l_returnflag")( + assertResult("l_orderkey,l_returnflag")( buckets.get.sortColumnNames .mkString(",")) - assertResult("l_orderkey")( + assertResult("l_partkey")( buckets.get.bucketColumnNames .mkString(",")) - assertResult("l_partkey,l_returnflag")( + assertResult("l_orderkey,l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get .mkString(",")) - assertResult("l_partkey")( + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption @@ -854,7 +853,7 @@ class GlutenClickHouseMergeTreePathBasedWriteSuite .flatMap(partition => partition.asInstanceOf[GlutenMergeTreePartition].partList) .map(_.name) .distinct - assertResult(1)(touchedParts.size) + assertResult(4)(touchedParts.size) // test upsert on partitioned & bucketed table upsertSourceTableAndCheck(dataPath) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala index 70c6553416e2..27bd4372aa64 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteSuite.scala @@ -434,9 +434,6 @@ class GlutenClickHouseMergeTreeWriteSuite val df1 = spark.sql(s""" | delete from lineitem_mergetree_delete where l_orderkey = 12647 |""".stripMargin) -// assert( -// df1.collect().apply(0).get(0) == 1 -// ) { val df = spark.sql(s""" @@ -868,8 +865,8 @@ class GlutenClickHouseMergeTreeWriteSuite |) |USING clickhouse |PARTITIONED BY (l_shipdate) - |CLUSTERED BY (l_orderkey) - |${if (sparkVersion.equals("3.2")) "" else "SORTED BY (l_partkey, l_returnflag)"} INTO 4 BUCKETS + |CLUSTERED BY (l_partkey) + |${if (sparkVersion.equals("3.2")) "" else "SORTED BY (l_orderkey, l_returnflag)"} INTO 4 BUCKETS |LOCATION '$basePath/lineitem_mergetree_bucket' |""".stripMargin) @@ -919,7 +916,7 @@ class GlutenClickHouseMergeTreeWriteSuite if (sparkVersion.equals("3.2")) { assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).orderByKeyOption.isEmpty) } else { - assertResult("l_partkey,l_returnflag")( + assertResult("l_orderkey,l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption @@ -956,7 +953,7 @@ class GlutenClickHouseMergeTreeWriteSuite .flatMap(partition => partition.asInstanceOf[GlutenMergeTreePartition].partList) .map(_.name) .distinct - assertResult(1)(touchedParts.size) + assertResult(4)(touchedParts.size) // test upsert on partitioned & bucketed table upsertSourceTableAndCheck("lineitem_mergetree_bucket") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetBucketSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetBucketSuite.scala index 14d3e0130b4c..c164fae708f8 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetBucketSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetBucketSuite.scala @@ -627,7 +627,7 @@ class GlutenClickHouseTPCHParquetBucketSuite } val touchedBuckets = scanExec.head.getPartitions .flatMap(partition => partition.asInstanceOf[FilePartition].files) - .flatMap(f => BucketingUtils.getBucketId(new Path(f.filePath).getName)) + .flatMap(f => BucketingUtils.getBucketId(new Path(f.filePath.toString()).getName)) .distinct // two files from part0-0,part0-1,part1-0,part1-1 assert(touchedBuckets.size == 1) diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CustomSum.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CustomSum.scala index c2d7cf622b3d..be361277fb1e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CustomSum.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CustomSum.scala @@ -21,7 +21,6 @@ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.trees.TreePattern.{SUM, TreePattern} import org.apache.spark.sql.catalyst.trees.UnaryLike -import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -43,8 +42,7 @@ case class CustomSum(child: Expression, failOnError: Boolean = SQLConf.get.ansiE override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(NumericType, YearMonthIntervalType, DayTimeIntervalType)) - override def checkInputDataTypes(): TypeCheckResult = - TypeUtils.checkForAnsiIntervalOrNumericType(child.dataType, "sum") + override def checkInputDataTypes(): TypeCheckResult = TypeCheckResult.TypeCheckSuccess final override val nodePatterns: Seq[TreePattern] = Seq(SUM) @@ -141,11 +139,7 @@ case class CustomSum(child: Expression, failOnError: Boolean = SQLConf.get.ansiE * overflow has happened. So now, if ansi is enabled, then throw exception, if not then return * null. If sum is not null, then return the sum. */ - override lazy val evaluateExpression: Expression = resultType match { - case d: DecimalType => - If(isEmpty, Literal.create(null, resultType), CheckOverflowInSum(sum, d, !failOnError)) - case _ => sum - } + override lazy val evaluateExpression: Expression = sum override protected def withNewChildInternal(newChild: Expression): CustomSum = copy(child = newChild) diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala index d9d356b67ada..bd1665330a89 100644 --- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala +++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala @@ -28,28 +28,27 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.catalyst.expressions.{Attribute, BinaryExpression, Expression} -import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate} +import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.plans.physical.Distribution -import org.apache.spark.sql.catalyst.plans.physical.Partitioning +import org.apache.spark.sql.catalyst.plans.physical.{Distribution, Partitioning} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.trees.TernaryLike import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{InputPartition, Scan} -import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, GlobalLimitExec, SparkPlan, TakeOrderedAndProjectExec} -import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec +import org.apache.spark.sql.execution.{FileSourceScanExec, GlobalLimitExec, SparkPlan, TakeOrderedAndProjectExec} import org.apache.spark.sql.execution.command.DataWritingCommandExec -import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, HadoopFsRelation, PartitionDirectory, PartitionedFile, PartitioningAwareFileIndex, WriteJobDescription, WriteTaskResult} -import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ShuffleExchangeLike} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.parquet.schema.MessageType import java.util.{ArrayList => JArrayList, Map => JMap} @@ -244,4 +243,9 @@ trait SparkShims { def dateTimestampFormatInReadIsDefaultValue(csvOptions: CSVOptions, timeZone: String): Boolean def isPlannedV1Write(write: DataWritingCommandExec): Boolean = false + + def createParquetFilters( + conf: SQLConf, + schema: MessageType, + caseSensitive: Option[Boolean] = None): ParquetFilters } diff --git a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala index 22122c5837dc..b9c37ef3d730 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala @@ -33,20 +33,25 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{Distribution, HashClusteredDistribution} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.{FileSourceScanExec, PartitionedFileUtil, SparkPlan} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.FileFormatWriter.Empty2Null +import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.parquet.schema.MessageType import java.util.{HashMap => JHashMap, Map => JMap} @@ -250,4 +255,20 @@ class Spark32Shims extends SparkShims { csvOptions.dateFormat == default.dateFormat && csvOptions.timestampFormat == default.timestampFormat } + + override def createParquetFilters( + conf: SQLConf, + schema: MessageType, + caseSensitive: Option[Boolean] = None): ParquetFilters = { + new ParquetFilters( + schema, + conf.parquetFilterPushDownDate, + conf.parquetFilterPushDownTimestamp, + conf.parquetFilterPushDownDecimal, + conf.parquetFilterPushDownStringStartWith, + conf.parquetFilterPushDownInFilterThreshold, + caseSensitive.getOrElse(conf.caseSensitiveAnalysis), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED) + ) + } } diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala index fdc782484e02..d6292b46c261 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala @@ -34,21 +34,26 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec import org.apache.spark.sql.catalyst.util.TimestampFormatter import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.execution.{FileSourceScanExec, PartitionedFileUtil, SparkPlan} import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.FileFormatWriter.Empty2Null +import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.parquet.schema.MessageType import java.time.ZoneOffset import java.util.{HashMap => JHashMap, Map => JMap} @@ -341,4 +346,20 @@ class Spark33Shims extends SparkShims { csvOptions.timestampFormatInRead == default.timestampFormatInRead && csvOptions.timestampNTZFormatInRead == default.timestampNTZFormatInRead } + + override def createParquetFilters( + conf: SQLConf, + schema: MessageType, + caseSensitive: Option[Boolean] = None): ParquetFilters = { + new ParquetFilters( + schema, + conf.parquetFilterPushDownDate, + conf.parquetFilterPushDownTimestamp, + conf.parquetFilterPushDownDecimal, + conf.parquetFilterPushDownStringStartWith, + conf.parquetFilterPushDownInFilterThreshold, + caseSensitive.getOrElse(conf.caseSensitiveAnalysis), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED) + ) + } } diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 171d412389be..c718f4ed25d6 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -36,22 +36,26 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution, KeyGroupedPartitioning, Partitioning} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, InternalRowComparableWrapper, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ +import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.parquet.schema.MessageType import java.time.ZoneOffset import java.util.{HashMap => JHashMap, Map => JMap} @@ -468,4 +472,20 @@ class Spark34Shims extends SparkShims { override def isPlannedV1Write(write: DataWritingCommandExec): Boolean = { write.cmd.isInstanceOf[V1WriteCommand] && SQLConf.get.plannedWriteEnabled } + + override def createParquetFilters( + conf: SQLConf, + schema: MessageType, + caseSensitive: Option[Boolean] = None): ParquetFilters = { + new ParquetFilters( + schema, + conf.parquetFilterPushDownDate, + conf.parquetFilterPushDownTimestamp, + conf.parquetFilterPushDownDecimal, + conf.parquetFilterPushDownStringPredicate, + conf.parquetFilterPushDownInFilterThreshold, + caseSensitive.getOrElse(conf.caseSensitiveAnalysis), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED) + ) + } } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 142403ada099..95571f166ebe 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -36,24 +36,26 @@ import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Dist import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, InternalRowComparableWrapper, TimestampFormatter} +import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Scan} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetRowIndexUtil} +import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters, ParquetRowIndexUtil} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ShuffleExchangeLike} import org.apache.spark.sql.execution.window.{WindowGroupLimitExec, WindowGroupLimitExecShim} -import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.storage.{BlockId, BlockManagerId} import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.parquet.schema.MessageType import java.time.ZoneOffset import java.util.{HashMap => JHashMap, Map => JMap} @@ -493,4 +495,20 @@ class Spark35Shims extends SparkShims { override def isPlannedV1Write(write: DataWritingCommandExec): Boolean = { write.cmd.isInstanceOf[V1WriteCommand] && SQLConf.get.plannedWriteEnabled } + + override def createParquetFilters( + conf: SQLConf, + schema: MessageType, + caseSensitive: Option[Boolean] = None): ParquetFilters = { + new ParquetFilters( + schema, + conf.parquetFilterPushDownDate, + conf.parquetFilterPushDownTimestamp, + conf.parquetFilterPushDownDecimal, + conf.parquetFilterPushDownStringPredicate, + conf.parquetFilterPushDownInFilterThreshold, + caseSensitive.getOrElse(conf.caseSensitiveAnalysis), + RebaseSpec(LegacyBehaviorPolicy.CORRECTED) + ) + } } From 8f214843ae09ae5d72f150470b03f7e1113e0234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Fri, 14 Jun 2024 13:47:17 +0800 Subject: [PATCH 270/402] [VL] Small change to always use testGluten instead of legacy old way (#6075) --- .../execution/datasources/GlutenFileMetadataStructSuite.scala | 3 +-- .../execution/datasources/GlutenFileMetadataStructSuite.scala | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index 5e4d1ed8f81f..efa0fbae062b 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -21,7 +21,6 @@ import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType} @@ -60,7 +59,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS f: (DataFrame, Map[String, Any], Map[String, Any]) => Unit): Unit = { Seq("parquet").foreach { testFileFormat => - test(s"$GLUTEN_TEST metadata struct ($testFileFormat): " + testName) { + testGluten(s"metadata struct ($testFileFormat): " + testName) { withTempDir { dir => import scala.collection.JavaConverters._ diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index 1d35c8656de1..b3b9ea7393c3 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -21,7 +21,6 @@ import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType} @@ -61,7 +60,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS f: (DataFrame, Map[String, Any], Map[String, Any]) => Unit): Unit = { Seq("parquet").foreach { testFileFormat => - test(s"$GLUTEN_TEST metadata struct ($testFileFormat): " + testName) { + testGluten(s"metadata struct ($testFileFormat): " + testName) { withTempDir { dir => import scala.collection.JavaConverters._ From c1deb127a4f818866ecd311168688af4215aadd5 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 14 Jun 2024 15:42:33 +0800 Subject: [PATCH 271/402] [DOC] Document how to use cmake-format in vs code (#6089) --- docs/developers/NewToGluten.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index a397003adf36..54f379148d39 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -289,6 +289,26 @@ Search `default formatter` in `Settings`, select Clang-Format. If your formatOnSave still make no effect, you can use shortcut `SHIFT+ALT+F` to format one file manually. +### Cmake format + +To format cmake files, like CMakeLists.txt & *.cmake, please install `cmake-format`. +``` +pip3 install --user cmake-format +``` +Here is an example to format a file in command line. +``` +cmake-format --first-comment-is-literal True --in-place cpp/velox/CMakeLists.txt +``` + +After the above installation, you can optionally do some configuration in Visual Studio Code to easily format cmake files. +1. Install `cmake-format` extension in Visual Studio Code. +2. Configure the extension. To do this, open the settings (File -> Preferences -> Settings), search for `cmake-format`, + and do the below settings: + * Set Args: `--first-comment-is-literal=True`. + * Set Exe Path to the path of the `cmake-format` command. If you installed `cmake-format` in a standard + location, you might not need to change this setting. +3. Now, you can format your CMake files by right-clicking in a file and selecting `Format Document`. + # Debug cpp code with coredump ```bash From 1cafa2826e428479c0ccca94d7f16470ff5eea8a Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 16:45:31 +0800 Subject: [PATCH 272/402] [VL] Minor refactors on ColumnarRuleApplier (#6086) --- .../columnar/ColumnarRuleApplier.scala | 36 +++++++++++++ .../enumerated/EnumeratedApplier.scala | 44 +++++----------- .../columnar/heuristic/HeuristicApplier.scala | 52 +++++++------------ .../apache/spark/sql/SparkQueryRunner.scala | 39 ++++++++------ 4 files changed, 91 insertions(+), 80 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala index 17bf017305f2..ee5bcd883e7e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala @@ -16,8 +16,44 @@ */ package org.apache.gluten.extension.columnar +import org.apache.gluten.GlutenConfig +import org.apache.gluten.metrics.GlutenTimeMetric +import org.apache.gluten.utils.LogLevelUtil + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor} import org.apache.spark.sql.execution.SparkPlan trait ColumnarRuleApplier { def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan } + +object ColumnarRuleApplier { + class Executor(phase: String, rules: Seq[Rule[SparkPlan]]) extends RuleExecutor[SparkPlan] { + private val batch: Batch = + Batch(s"Columnar (Phase [$phase])", Once, rules.map(r => new LoggedRule(r)): _*) + + // TODO Remove this exclusion then pass Spark's idempotence check. + override protected val excludedOnceBatches: Set[String] = Set(batch.name) + + override protected def batches: Seq[Batch] = List(batch) + } + + private class LoggedRule(delegate: Rule[SparkPlan]) + extends Rule[SparkPlan] + with Logging + with LogLevelUtil { + // Columnar plan change logging added since https://github.com/apache/incubator-gluten/pull/456. + private val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel + override val ruleName: String = delegate.ruleName + + override def apply(plan: SparkPlan): SparkPlan = GlutenTimeMetric.withMillisTime { + logOnLevel( + transformPlanLogLevel, + s"Preparing to apply rule $ruleName on plan:\n${plan.toString}") + val out = delegate.apply(plan) + logOnLevel(transformPlanLogLevel, s"Plan after applied rule $ruleName:\n${plan.toString}") + out + }(t => logOnLevel(transformPlanLogLevel, s"Applying rule $ruleName took $t ms.")) + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index 26201dc1baa3..d5260f66adba 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -22,13 +22,12 @@ import org.apache.gluten.extension.columnar._ import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, RewriteSubqueryBroadcast} import org.apache.gluten.extension.columnar.transition.{InsertTransitions, RemoveTransitions} import org.apache.gluten.extension.columnar.util.AdaptiveContext -import org.apache.gluten.metrics.GlutenTimeMetric import org.apache.gluten.utils.{LogLevelUtil, PhysicalPlanSelector} import org.apache.spark.annotation.Experimental import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.rules.{PlanChangeLogger, Rule} +import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{ColumnarCollapseTransformStages, GlutenFallbackReporter, SparkPlan} import org.apache.spark.util.SparkRuleUtil @@ -47,41 +46,26 @@ class EnumeratedApplier(session: SparkSession) with LogLevelUtil { // An empirical value. private val aqeStackTraceIndex = 16 - - private lazy val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel - private lazy val planChangeLogger = new PlanChangeLogger[SparkPlan]() - private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = PhysicalPlanSelector.maybe(session, plan) { - val transformed = transformPlan(transformRules(outputsColumnar), plan, "transform") + val transformed = + transformPlan("transform", transformRules(outputsColumnar).map(_(session)), plan) val postPlan = maybeAqe { - transformPlan(postRules(), transformed, "post") + transformPlan("post", postRules().map(_(session)), transformed) } - val finalPlan = transformPlan(finalRules(), postPlan, "final") + val finalPlan = transformPlan("final", finalRules().map(_(session)), postPlan) finalPlan } private def transformPlan( - getRules: List[SparkSession => Rule[SparkPlan]], - plan: SparkPlan, - step: String) = GlutenTimeMetric.withMillisTime { - logOnLevel( - transformPlanLogLevel, - s"${step}ColumnarTransitions preOverriden plan:\n${plan.toString}") - val overridden = getRules.foldLeft(plan) { - (p, getRule) => - val rule = getRule(session) - val newPlan = rule(p) - planChangeLogger.logRule(rule.ruleName, p, newPlan) - newPlan - } - logOnLevel( - transformPlanLogLevel, - s"${step}ColumnarTransitions afterOverriden plan:\n${overridden.toString}") - overridden - }(t => logOnLevel(transformPlanLogLevel, s"${step}Transform SparkPlan took: $t ms.")) + phase: String, + rules: Seq[Rule[SparkPlan]], + plan: SparkPlan): SparkPlan = { + val executor = new ColumnarRuleApplier.Executor(phase, rules) + executor.execute(plan) + } private def maybeAqe[T](f: => T): T = { adaptiveContext.setAdaptiveContext() @@ -96,7 +80,7 @@ class EnumeratedApplier(session: SparkSession) * Rules to let planner create a suggested Gluten plan being sent to `fallbackPolicies` in which * the plan will be breakdown and decided to be fallen back or not. */ - private def transformRules(outputsColumnar: Boolean): List[SparkSession => Rule[SparkPlan]] = { + private def transformRules(outputsColumnar: Boolean): Seq[SparkSession => Rule[SparkPlan]] = { List( (_: SparkSession) => RemoveTransitions, (spark: SparkSession) => FallbackOnANSIMode(spark), @@ -126,7 +110,7 @@ class EnumeratedApplier(session: SparkSession) * Rules applying to non-fallen-back Gluten plans. To do some post cleanup works on the plan to * make sure it be able to run and be compatible with Spark's execution engine. */ - private def postRules(): List[SparkSession => Rule[SparkPlan]] = + private def postRules(): Seq[SparkSession => Rule[SparkPlan]] = List( (s: SparkSession) => RemoveTopmostColumnarToRow(s, adaptiveContext.isAdaptiveContext())) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarPostRules() ::: @@ -137,7 +121,7 @@ class EnumeratedApplier(session: SparkSession) * Rules consistently applying to all input plans after all other rules have been applied, despite * whether the input plan is fallen back or not. */ - private def finalRules(): List[SparkSession => Rule[SparkPlan]] = { + private def finalRules(): Seq[SparkSession => Rule[SparkPlan]] = { List( // The rule is required despite whether the stage is fallen back or not. Since // ColumnarCachedBatchSerializer is statically registered to Spark without a columnar rule diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index eb5c561bfa8d..ad68786e6579 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -23,12 +23,11 @@ import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTable import org.apache.gluten.extension.columnar.rewrite.RewriteSparkPlanRulesManager import org.apache.gluten.extension.columnar.transition.{InsertTransitions, RemoveTransitions} import org.apache.gluten.extension.columnar.util.AdaptiveContext -import org.apache.gluten.metrics.GlutenTimeMetric import org.apache.gluten.utils.{LogLevelUtil, PhysicalPlanSelector} import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.rules.{PlanChangeLogger, Rule} +import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.{ColumnarCollapseTransformStages, GlutenFallbackReporter, SparkPlan} import org.apache.spark.util.SparkRuleUtil @@ -42,54 +41,39 @@ class HeuristicApplier(session: SparkSession) with LogLevelUtil { // This is an empirical value, may need to be changed for supporting other versions of spark. private val aqeStackTraceIndex = 19 - - private lazy val transformPlanLogLevel = GlutenConfig.getConf.transformPlanLogLevel - private lazy val planChangeLogger = new PlanChangeLogger[SparkPlan]() - private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) - override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = + override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = { withTransformRules(transformRules(outputsColumnar)).apply(plan) + } // Visible for testing. - def withTransformRules(transformRules: List[SparkSession => Rule[SparkPlan]]): Rule[SparkPlan] = + def withTransformRules(transformRules: Seq[SparkSession => Rule[SparkPlan]]): Rule[SparkPlan] = plan => PhysicalPlanSelector.maybe(session, plan) { val finalPlan = prepareFallback(plan) { p => - val suggestedPlan = transformPlan(transformRules, p, "transform") - transformPlan(fallbackPolicies(), suggestedPlan, "fallback") match { + val suggestedPlan = transformPlan("transform", transformRules.map(_(session)), p) + transformPlan("fallback", fallbackPolicies().map(_(session)), suggestedPlan) match { case FallbackNode(fallbackPlan) => // we should use vanilla c2r rather than native c2r, // and there should be no `GlutenPlan` any more, // so skip the `postRules()`. fallbackPlan case plan => - transformPlan(postRules(), plan, "post") + transformPlan("post", postRules().map(_(session)), plan) } } - transformPlan(finalRules(), finalPlan, "final") + transformPlan("final", finalRules().map(_(session)), finalPlan) } private def transformPlan( - getRules: List[SparkSession => Rule[SparkPlan]], - plan: SparkPlan, - step: String) = GlutenTimeMetric.withMillisTime { - logOnLevel( - transformPlanLogLevel, - s"${step}ColumnarTransitions preOverridden plan:\n${plan.toString}") - val overridden = getRules.foldLeft(plan) { - (p, getRule) => - val rule = getRule(session) - val newPlan = rule(p) - planChangeLogger.logRule(rule.ruleName, p, newPlan) - newPlan - } - logOnLevel( - transformPlanLogLevel, - s"${step}ColumnarTransitions afterOverridden plan:\n${overridden.toString}") - overridden - }(t => logOnLevel(transformPlanLogLevel, s"${step}Transform SparkPlan took: $t ms.")) + phase: String, + rules: Seq[Rule[SparkPlan]], + plan: SparkPlan): SparkPlan = { + val executor = new ColumnarRuleApplier.Executor(phase, rules) + executor.execute(plan) + } private def prepareFallback[T](plan: SparkPlan)(f: SparkPlan => T): T = { adaptiveContext.setAdaptiveContext() @@ -106,7 +90,7 @@ class HeuristicApplier(session: SparkSession) * Rules to let planner create a suggested Gluten plan being sent to `fallbackPolicies` in which * the plan will be breakdown and decided to be fallen back or not. */ - private def transformRules(outputsColumnar: Boolean): List[SparkSession => Rule[SparkPlan]] = { + private def transformRules(outputsColumnar: Boolean): Seq[SparkSession => Rule[SparkPlan]] = { List( (_: SparkSession) => RemoveTransitions, (spark: SparkSession) => FallbackOnANSIMode(spark), @@ -138,7 +122,7 @@ class HeuristicApplier(session: SparkSession) * Rules to add wrapper `FallbackNode`s on top of the input plan, as hints to make planner fall * back the whole input plan to the original vanilla Spark plan. */ - private def fallbackPolicies(): List[SparkSession => Rule[SparkPlan]] = { + private def fallbackPolicies(): Seq[SparkSession => Rule[SparkPlan]] = { List( (_: SparkSession) => ExpandFallbackPolicy(adaptiveContext.isAdaptiveContext(), adaptiveContext.originalPlan())) @@ -148,7 +132,7 @@ class HeuristicApplier(session: SparkSession) * Rules applying to non-fallen-back Gluten plans. To do some post cleanup works on the plan to * make sure it be able to run and be compatible with Spark's execution engine. */ - private def postRules(): List[SparkSession => Rule[SparkPlan]] = + private def postRules(): Seq[SparkSession => Rule[SparkPlan]] = List( (s: SparkSession) => RemoveTopmostColumnarToRow(s, adaptiveContext.isAdaptiveContext())) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarPostRules() ::: @@ -159,7 +143,7 @@ class HeuristicApplier(session: SparkSession) * Rules consistently applying to all input plans after all other rules have been applied, despite * whether the input plan is fallen back or not. */ - private def finalRules(): List[SparkSession => Rule[SparkPlan]] = { + private def finalRules(): Seq[SparkSession => Rule[SparkPlan]] = { List( // The rule is required despite whether the stage is fallen back or not. Since // ColumnarCachedBatchSerializer is statically registered to Spark without a columnar rule diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala index bb11a679f9eb..b68f74c1d5ed 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkQueryRunner.scala @@ -18,13 +18,9 @@ package org.apache.spark.sql import org.apache.spark.{SparkContext, Success, TaskKilled} import org.apache.spark.executor.ExecutorMetrics -import org.apache.spark.scheduler.{ - SparkListener, - SparkListenerExecutorMetricsUpdate, - SparkListenerTaskEnd, - SparkListenerTaskStart -} +import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorMetricsUpdate, SparkListenerTaskEnd, SparkListenerTaskStart} import org.apache.spark.sql.KillTaskListener.INIT_WAIT_TIME_MS +import org.apache.spark.sql.catalyst.QueryPlanningTracker import com.google.common.base.Preconditions import org.apache.commons.lang3.RandomUtils @@ -50,7 +46,8 @@ object SparkQueryRunner { "ProcessTreePythonVMemory", "ProcessTreePythonRSSMemory", "ProcessTreeOtherVMemory", - "ProcessTreeOtherRSSMemory") + "ProcessTreeOtherRSSMemory" + ) def runQuery( spark: SparkSession, @@ -82,25 +79,33 @@ object SparkQueryRunner { println(s"Executing SQL query from resource path $queryPath...") try { + val tracker = new QueryPlanningTracker val sql = resourceToString(queryPath) val prev = System.nanoTime() val df = spark.sql(sql) - val rows = df.collect() + val rows = QueryPlanningTracker.withTracker(tracker) { + df.collect() + } if (explain) { df.explain(extended = true) } - val planMillis = - df.queryExecution.tracker.phases.values.map(p => p.endTimeMs - p.startTimeMs).sum + val sparkTracker = df.queryExecution.tracker + val sparkRulesMillis = + sparkTracker.rules.map(_._2.totalTimeNs).sum / 1000000L + val otherRulesMillis = + tracker.rules.map(_._2.totalTimeNs).sum / 1000000L + val planMillis = sparkRulesMillis + otherRulesMillis val totalMillis = (System.nanoTime() - prev) / 1000000L val collectedMetrics = metrics.map(name => (name, em.getMetricValue(name))).toMap RunResult(rows, planMillis, totalMillis - planMillis, collectedMetrics) } finally { sc.removeSparkListener(metricsListener) - killTaskListener.foreach(l => { - sc.removeSparkListener(l) - println(s"Successful kill rate ${"%.2f%%" - .format(100 * l.successfulKillRate())} during execution of app: ${sc.applicationId}") - }) + killTaskListener.foreach( + l => { + sc.removeSparkListener(l) + println(s"Successful kill rate ${"%.2f%%" + .format(100 * l.successfulKillRate())} during execution of app: ${sc.applicationId}") + }) sc.setJobDescription(null) } } @@ -166,7 +171,8 @@ class KillTaskListener(val sc: SparkContext) extends SparkListener { val total = Math.min( stageKillMaxWaitTimeLookup.computeIfAbsent(taskStart.stageId, _ => Long.MaxValue), stageKillWaitTimeLookup - .computeIfAbsent(taskStart.stageId, _ => INIT_WAIT_TIME_MS)) + .computeIfAbsent(taskStart.stageId, _ => INIT_WAIT_TIME_MS) + ) val elapsed = System.currentTimeMillis() - startMs val remaining = total - elapsed if (remaining <= 0L) { @@ -180,6 +186,7 @@ class KillTaskListener(val sc: SparkContext) extends SparkListener { } throw new IllegalStateException() } + val elapsed = wait() // We have 50% chance to kill the task. FIXME make it configurable? From 284b304a9aec28c506fbb69a3c8393125ff0bac2 Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Fri, 14 Jun 2024 14:19:19 +0530 Subject: [PATCH 273/402] [GLUTEN-6026][VL] Add Support for HiveFileFormat parquet write for Spark 3.4+ (#6062) --- .../backendsapi/velox/VeloxBackend.scala | 38 +++++++++++++++++-- .../VeloxParquetWriteForHiveSuite.scala | 6 +-- docs/velox-backend-limitations.md | 4 ++ .../org/apache/gluten/GlutenConfig.scala | 12 ++++++ 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 6bc7df98cca2..158be10f486c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand import org.apache.spark.sql.execution.datasources.{FileFormat, InsertIntoHadoopFsRelationCommand} import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.hive.execution.HiveFileFormat import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -182,6 +183,30 @@ object VeloxBackendSettings extends BackendSettingsApi { bucketSpec: Option[BucketSpec], options: Map[String, String]): ValidationResult = { + // Validate if HiveFileFormat write is supported based on output file type + def validateHiveFileFormat(hiveFileFormat: HiveFileFormat): Option[String] = { + // Reflect to get access to fileSinkConf which contains the output file format + val fileSinkConfField = format.getClass.getDeclaredField("fileSinkConf") + fileSinkConfField.setAccessible(true) + val fileSinkConf = fileSinkConfField.get(hiveFileFormat) + val tableInfoField = fileSinkConf.getClass.getDeclaredField("tableInfo") + tableInfoField.setAccessible(true) + val tableInfo = tableInfoField.get(fileSinkConf) + val getOutputFileFormatClassNameMethod = tableInfo.getClass + .getDeclaredMethod("getOutputFileFormatClassName") + val outputFileFormatClassName = getOutputFileFormatClassNameMethod.invoke(tableInfo) + + // Match based on the output file format class name + outputFileFormatClassName match { + case "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" => + None + case _ => + Some( + "HiveFileFormat is supported only with Parquet as the output file type" + ) // Unsupported format + } + } + def validateCompressionCodec(): Option[String] = { // Velox doesn't support brotli and lzo. val unSupportedCompressions = Set("brotli", "lzo", "lz4raw", "lz4_raw") @@ -194,7 +219,7 @@ object VeloxBackendSettings extends BackendSettingsApi { } // Validate if all types are supported. - def validateDateTypes(): Option[String] = { + def validateDataTypes(): Option[String] = { val unsupportedTypes = fields.flatMap { field => field.dataType match { @@ -222,8 +247,13 @@ object VeloxBackendSettings extends BackendSettingsApi { def validateFileFormat(): Option[String] = { format match { - case _: ParquetFileFormat => None - case _: FileFormat => Some("Only parquet fileformat is supported in Velox backend.") + case _: ParquetFileFormat => None // Parquet is directly supported + case h: HiveFileFormat if GlutenConfig.getConf.enableHiveFileFormatWriter => + validateHiveFileFormat(h) // Parquet via Hive SerDe + case _ => + Some( + "Only ParquetFileFormat and HiveFileFormat are supported." + ) // Unsupported format } } @@ -250,7 +280,7 @@ object VeloxBackendSettings extends BackendSettingsApi { validateCompressionCodec() .orElse(validateFileFormat()) .orElse(validateFieldMetadata()) - .orElse(validateDateTypes()) + .orElse(validateDataTypes()) .orElse(validateWriteFilesOptions()) .orElse(validateBucketSpec()) match { case Some(reason) => ValidationResult.notOk(reason) diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala index 9597e3110a10..731f5ef4845c 100644 --- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala +++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala @@ -139,11 +139,7 @@ class VeloxParquetWriteForHiveSuite extends GlutenQueryTest with SQLTestUtils { withTable("t") { spark.sql("CREATE TABLE t (c int) STORED AS PARQUET") withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") { - if (isSparkVersionGE("3.4")) { - checkNativeWrite("INSERT OVERWRITE TABLE t SELECT 1 as c", checkNative = false) - } else { - checkNativeWrite("INSERT OVERWRITE TABLE t SELECT 1 as c", checkNative = true) - } + checkNativeWrite("INSERT OVERWRITE TABLE t SELECT 1 as c", checkNative = true) } checkAnswer(spark.table("t"), Row(1)) } diff --git a/docs/velox-backend-limitations.md b/docs/velox-backend-limitations.md index 75b52f38e17a..002bbb3c3017 100644 --- a/docs/velox-backend-limitations.md +++ b/docs/velox-backend-limitations.md @@ -118,6 +118,10 @@ spark.range(100).toDF("id") .saveAsTable("velox_ctas") ``` +#### HiveFileFormat write + +Gluten supports writes of HiveFileFormat when the output file type is of type `parquet` only + #### NaN support Velox does NOT support NaN. So unexpected result can be obtained for a few cases, e.g., comparing a number with NaN. diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 13ad8e47113b..a4e5a4425e3b 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -438,6 +438,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { def dynamicOffHeapSizingEnabled: Boolean = conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) + + def enableHiveFileFormatWriter: Boolean = conf.getConf(NATIVE_HIVEFILEFORMAT_WRITER_ENABLED) } object GlutenConfig { @@ -1578,6 +1580,16 @@ object GlutenConfig { .booleanConf .createOptional + val NATIVE_HIVEFILEFORMAT_WRITER_ENABLED = + buildConf("spark.gluten.sql.native.hive.writer.enabled") + .internal() + .doc( + "This is config to specify whether to enable the native columnar writer for " + + "HiveFileFormat. Currently only supports HiveFileFormat with Parquet as the output " + + "file type.") + .booleanConf + .createWithDefault(true) + val NATIVE_ARROW_READER_ENABLED = buildConf("spark.gluten.sql.native.arrow.reader.enabled") .internal() From a36d3077d0acd7a9cc73a6ad683d62e5bb49cbba Mon Sep 17 00:00:00 2001 From: Chang chen Date: Fri, 14 Jun 2024 21:12:43 +0800 Subject: [PATCH 274/402] [GLUTEN-6091][CH] Avoid using LD_PRELOAD in child process (#6092) [CH] Avoid using LD_PRELOAD in child process --- cpp-ch/local-engine/Common/CHUtil.cpp | 3 +++ cpp-ch/local-engine/Functions/CMakeLists.txt | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index fa6124cf011f..2dd5f67687db 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -894,6 +894,9 @@ void BackendInitializerUtil::init(std::string * plan) cleanup_threads, 0, // We don't need any threads one all the parts will be deleted cleanup_threads); + + // Avoid using LD_PRELOAD in child process + unsetenv("LD_PRELOAD"); }); } diff --git a/cpp-ch/local-engine/Functions/CMakeLists.txt b/cpp-ch/local-engine/Functions/CMakeLists.txt index 74697315597b..9e31595e03d5 100644 --- a/cpp-ch/local-engine/Functions/CMakeLists.txt +++ b/cpp-ch/local-engine/Functions/CMakeLists.txt @@ -59,6 +59,10 @@ if(TARGET ch_rust::blake3) list(APPEND PRIVATE_LIBS ch_rust::blake3) endif() +if(TARGET ch_contrib::gwp_asan) + list(APPEND PRIVATE_LIBS ch_contrib::gwp_asan) +endif() + list(APPEND OBJECT_LIBS $) target_link_libraries(gluten_spark_functions_obj PRIVATE ${PRIVATE_LIBS}) From a08a57c61f9971cf2ad1b99cd48f275bf0c78c7d Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Sun, 16 Jun 2024 03:58:01 -0500 Subject: [PATCH 275/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240616) (#6100) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240616) * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/64412 * Fix UT due to https://github.com/ClickHouse/ClickHouse/pull/64986 * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/60556 * Ignore TPCH Q17 which include avg * Fix 'Invalid Field get from type Decimal64 to type Int64' in debug build. --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- .../execution/GlutenClickHouseDecimalSuite.scala | 3 ++- cpp-ch/clickhouse.version | 4 ++-- cpp-ch/local-engine/Common/CHUtil.cpp | 3 +++ .../registerGlutenDiskObjectStorage.cpp | 13 +++++++++++-- cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp | 5 +++++ .../SubstraitSource/ExcelTextFormatFile.cpp | 7 ++++++- .../Storages/SubstraitSource/ReadBufferBuilder.cpp | 4 ++-- 7 files changed, 31 insertions(+), 8 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala index 3aa498ea35c1..892d2ff61855 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala @@ -332,11 +332,12 @@ class GlutenClickHouseDecimalSuite spark.sql("drop table if exists decimals_test") } } - + // FIXME: Support AVG for Decimal Type Seq("true", "false").foreach { allowPrecisionLoss => Range .inclusive(1, 22) + .filter(_ != 17) // Ignore Q17 which include avg .foreach { sql_num => { diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 991edb4f20e5..e374d3f5fd9e 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240612 -CH_COMMIT=e13cab114c5 +CH_BRANCH=rebase_ch/20240616 +CH_COMMIT=e0e4b947245 diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 2dd5f67687db..a4634c3f3bc7 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -761,6 +761,9 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) /// Initialize a dummy query cache. global_context->setQueryCache(0, 0, 0, 0); + + // We must set the application type to CLIENT to avoid ServerUUID::get() throw exception + global_context->setApplicationType(Context::ApplicationType::CLIENT); } } diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp index 800b51f93e94..c080e0525f3c 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp @@ -59,6 +59,14 @@ static S3::URI getS3URI( return uri; } +static std::string getEndpoint( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ContextPtr & context) +{ + return context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); +} + void registerGlutenS3ObjectStorage(ObjectStorageFactory & factory) { static constexpr auto disk_type = "s3_gluten"; @@ -74,8 +82,9 @@ void registerGlutenS3ObjectStorage(ObjectStorageFactory & factory) { auto uri = getS3URI(config, config_prefix, context); auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); - auto settings = getSettings(config, config_prefix, context); - auto client = getClient(config, config_prefix, context, *settings, true); + auto endpoint = getEndpoint(config, config_prefix, context); + auto settings = getSettings(config, config_prefix, context, endpoint, /* validate_settings */true); + auto client = getClient(endpoint, *settings, context, /* for_disk_s3 */true); auto key_generator = createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key); auto object_storage = std::make_shared( diff --git a/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp b/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp index 6f8df0ecbd75..2b4eb824a5fd 100644 --- a/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp +++ b/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp @@ -820,6 +820,11 @@ int64_t VariableLengthDataWriter::writeStruct(size_t row_idx, const DB::Tuple & auto v = field_value.get(); writer.unsafeWrite(reinterpret_cast(&v), buffer_address + offset + start + len_null_bitmap + i * 8); } + else if (writer.getWhichDataType().isDecimal64() || writer.getWhichDataType().isDateTime64()) + { + auto v = field_value.get(); + writer.unsafeWrite(reinterpret_cast(&v), buffer_address + offset + start + len_null_bitmap + i * 8); + } else writer.unsafeWrite( reinterpret_cast(&field_value.get()), buffer_address + offset + start + len_null_bitmap + i * 8); diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp index 038f280b0560..31ef5b9e1715 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp @@ -293,7 +293,12 @@ bool ExcelTextFormatReader::readField( return false; } - if (column_size == column.size()) + // See https://github.com/ClickHouse/ClickHouse/pull/60556 + // In case of failing to parse, we will always push element into nullmap. + // so, we need using nestedColumn to check if error occurs. + /// FIXME: move it to ExcelSerialization ??? + const auto nestedColumn = DB::removeNullable(column.getPtr()); + if (column_size == nestedColumn->size()) { skipErrorChars(*buf, has_quote, maybe_quote, escape, format_settings); column_back_func(column); diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp index d54ff985ec5f..ec967a869600 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -437,7 +437,7 @@ class S3FileReadBufferBuilder : public ReadBufferBuilder bucket, object.remote_path, "", - DB::S3Settings::RequestSettings(), + DB::S3::RequestSettings(), new_settings, /* use_external_buffer */ true, /* offset */ 0, From e2f90e96146b97c5fef1fad5fe704ce37faa9894 Mon Sep 17 00:00:00 2001 From: zhouyifan279 <88070094+zhouyifan279@users.noreply.github.com> Date: Mon, 17 Jun 2024 08:34:12 +0800 Subject: [PATCH 276/402] [GLUTEN-6072][BUILD] Fix permission denied error when install AWS SDK (#6073) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 49654d8a8cd5..d118ca0c9188 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -274,8 +274,8 @@ function setup_linux { local LINUX_VERSION_ID=$(. /etc/os-release && echo ${VERSION_ID}) # apply patches - sed -i 's/^ ninja -C "${BINARY_DIR}" install/ sudo ninja -C "${BINARY_DIR}" install/g' scripts/setup-helper-functions.sh sed -i 's/-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17/-march=native -std=c++17 -mno-avx512f/g' scripts/setup-helper-functions.sh + sed -i 's/SUDO="${SUDO:-""}"/SUDO="${SUDO:-"sudo --preserve-env"}"/g' scripts/setup-helper-functions.sh if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian" || "$LINUX_DISTRIBUTION" == "pop" ]]; then process_setup_ubuntu elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then From e4388e656175f1b4b01a2ade04ef07d397fe1e5a Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Mon, 17 Jun 2024 10:47:58 +0800 Subject: [PATCH 277/402] [VL] Enable sort-based shuffle in micro benchmark (#5942) --- cpp/core/CMakeLists.txt | 1 + cpp/core/jni/JniWrapper.cc | 18 +- cpp/core/shuffle/LocalPartitionWriter.cc | 3 +- cpp/core/shuffle/Partitioner.cc | 3 + cpp/core/shuffle/Partitioning.cc | 4 + cpp/core/shuffle/Partitioning.h | 2 +- cpp/core/shuffle/Payload.cc | 2 + cpp/core/shuffle/RandomPartitioner.cc | 56 +++ cpp/core/shuffle/RandomPartitioner.h | 48 ++ cpp/velox/CMakeLists.txt | 1 + cpp/velox/benchmarks/CMakeLists.txt | 2 - cpp/velox/benchmarks/GenericBenchmark.cc | 445 +++++++++++------- cpp/velox/benchmarks/ParquetWriteBenchmark.cc | 2 +- cpp/velox/benchmarks/ShuffleSplitBenchmark.cc | 394 ---------------- cpp/velox/benchmarks/common/BenchmarkUtils.cc | 23 +- cpp/velox/benchmarks/common/BenchmarkUtils.h | 13 +- .../benchmarks/common/FileReaderIterator.h | 2 - .../benchmarks/common/ParquetReaderIterator.h | 2 +- cpp/velox/compute/VeloxRuntime.cc | 22 +- cpp/velox/shuffle/VeloxShuffleWriter.cc | 42 ++ cpp/velox/shuffle/VeloxShuffleWriter.h | 9 + docs/developers/MicroBenchmarks.md | 101 +++- 22 files changed, 592 insertions(+), 603 deletions(-) create mode 100644 cpp/core/shuffle/RandomPartitioner.cc create mode 100644 cpp/core/shuffle/RandomPartitioner.h delete mode 100644 cpp/velox/benchmarks/ShuffleSplitBenchmark.cc create mode 100644 cpp/velox/shuffle/VeloxShuffleWriter.cc diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 3a4d6e9e8792..4d7c30402985 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -196,6 +196,7 @@ set(SPARK_COLUMNAR_PLUGIN_SRCS shuffle/Partitioning.cc shuffle/Payload.cc shuffle/rss/RssPartitionWriter.cc + shuffle/RandomPartitioner.cc shuffle/RoundRobinPartitioner.cc shuffle/ShuffleMemoryPool.cc shuffle/ShuffleReader.cc diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 4e069ec7a6d6..1e5326689229 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -756,6 +756,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe throw gluten::GlutenException(std::string("Short partitioning name can't be null")); } + // Build ShuffleWriterOptions. auto shuffleWriterOptions = ShuffleWriterOptions{ .bufferSize = bufferSize, .bufferReallocThreshold = reallocThreshold, @@ -763,7 +764,15 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe .taskAttemptId = (int64_t)taskAttemptId, .startPartitionId = startPartitionId, }; + auto shuffleWriterTypeC = env->GetStringUTFChars(shuffleWriterTypeJstr, JNI_FALSE); + auto shuffleWriterType = std::string(shuffleWriterTypeC); + env->ReleaseStringUTFChars(shuffleWriterTypeJstr, shuffleWriterTypeC); + + if (shuffleWriterType == "sort") { + shuffleWriterOptions.shuffleWriterType = kSortShuffle; + } + // Build PartitionWriterOptions. auto partitionWriterOptions = PartitionWriterOptions{ .mergeBufferSize = mergeBufferSize, .mergeThreshold = mergeThreshold, @@ -779,20 +788,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe partitionWriterOptions.codecBackend = getCodecBackend(env, codecBackendJstr); partitionWriterOptions.compressionMode = getCompressionMode(env, compressionModeJstr); } + std::unique_ptr partitionWriter; auto partitionWriterTypeC = env->GetStringUTFChars(partitionWriterTypeJstr, JNI_FALSE); auto partitionWriterType = std::string(partitionWriterTypeC); env->ReleaseStringUTFChars(partitionWriterTypeJstr, partitionWriterTypeC); - auto shuffleWriterTypeC = env->GetStringUTFChars(shuffleWriterTypeJstr, JNI_FALSE); - auto shuffleWriterType = std::string(shuffleWriterTypeC); - env->ReleaseStringUTFChars(shuffleWriterTypeJstr, shuffleWriterTypeC); - - if (shuffleWriterType == "sort") { - shuffleWriterOptions.shuffleWriterType = kSortShuffle; - } - if (partitionWriterType == "local") { if (dataFileJstr == NULL) { throw gluten::GlutenException(std::string("Shuffle DataFile can't be null")); diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc b/cpp/core/shuffle/LocalPartitionWriter.cc index 6c4a0af39a22..f56543bab5e0 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.cc +++ b/cpp/core/shuffle/LocalPartitionWriter.cc @@ -25,7 +25,6 @@ #include "shuffle/Payload.h" #include "shuffle/Spill.h" #include "shuffle/Utils.h" -#include "utils/Timer.h" namespace gluten { @@ -547,7 +546,7 @@ arrow::Status LocalPartitionWriter::evict( arrow::Status LocalPartitionWriter::evict(uint32_t partitionId, int64_t rawSize, const char* data, int64_t length) { rawPartitionLengths_[partitionId] += rawSize; - if (partitionId <= lastEvictPid_) { + if (partitionId < lastEvictPid_) { RETURN_NOT_OK(finishSpill()); } lastEvictPid_ = partitionId; diff --git a/cpp/core/shuffle/Partitioner.cc b/cpp/core/shuffle/Partitioner.cc index 80b4598a1f17..fb1a5aab44eb 100644 --- a/cpp/core/shuffle/Partitioner.cc +++ b/cpp/core/shuffle/Partitioner.cc @@ -18,6 +18,7 @@ #include "shuffle/Partitioner.h" #include "shuffle/FallbackRangePartitioner.h" #include "shuffle/HashPartitioner.h" +#include "shuffle/RandomPartitioner.h" #include "shuffle/RoundRobinPartitioner.h" #include "shuffle/SinglePartitioner.h" @@ -34,6 +35,8 @@ Partitioner::make(Partitioning partitioning, int32_t numPartitions, int32_t star return std::make_shared(); case Partitioning::kRange: return std::make_shared(numPartitions); + case Partitioning::kRandom: + return std::make_shared(numPartitions); default: return arrow::Status::Invalid("Unsupported partitioning type: " + std::to_string(partitioning)); } diff --git a/cpp/core/shuffle/Partitioning.cc b/cpp/core/shuffle/Partitioning.cc index dfe848d63046..84fe6ecd972f 100644 --- a/cpp/core/shuffle/Partitioning.cc +++ b/cpp/core/shuffle/Partitioning.cc @@ -23,6 +23,7 @@ static const std::string kSinglePartitioningName = "single"; static const std::string kRoundRobinPartitioningName = "rr"; static const std::string kHashPartitioningName = "hash"; static const std::string kRangePartitioningName = "range"; +static const std::string kRandomPartitioningName = "random"; } // namespace namespace gluten { @@ -39,6 +40,9 @@ Partitioning toPartitioning(std::string name) { if (name == kRangePartitioningName) { return Partitioning::kRange; } + if (name == kRandomPartitioningName) { + return Partitioning::kRandom; + } throw GlutenException("Invalid partition name: " + name); } diff --git a/cpp/core/shuffle/Partitioning.h b/cpp/core/shuffle/Partitioning.h index 1d65e9d6b993..a60d43561bee 100644 --- a/cpp/core/shuffle/Partitioning.h +++ b/cpp/core/shuffle/Partitioning.h @@ -20,7 +20,7 @@ #include namespace gluten { -enum Partitioning { kSingle, kRoundRobin, kHash, kRange }; +enum Partitioning { kSingle, kRoundRobin, kHash, kRange, kRandom /*for test only*/ }; Partitioning toPartitioning(std::string name); diff --git a/cpp/core/shuffle/Payload.cc b/cpp/core/shuffle/Payload.cc index beca3fa02d61..fb91c326b679 100644 --- a/cpp/core/shuffle/Payload.cc +++ b/cpp/core/shuffle/Payload.cc @@ -503,6 +503,7 @@ arrow::Status UncompressedDiskBlockPayload::serialize(arrow::io::OutputStream* o } arrow::Result> UncompressedDiskBlockPayload::readUncompressedBuffer() { + ScopedTimer timer(&writeTime_); readPos_++; int64_t bufferLength; RETURN_NOT_OK(inputStream_->Read(sizeof(int64_t), &bufferLength)); @@ -525,6 +526,7 @@ CompressedDiskBlockPayload::CompressedDiskBlockPayload( : Payload(Type::kCompressed, numRows, isValidityBuffer), inputStream_(inputStream), rawSize_(rawSize) {} arrow::Status CompressedDiskBlockPayload::serialize(arrow::io::OutputStream* outputStream) { + ScopedTimer timer(&writeTime_); ARROW_ASSIGN_OR_RAISE(auto block, inputStream_->Read(rawSize_)); RETURN_NOT_OK(outputStream->Write(block)); return arrow::Status::OK(); diff --git a/cpp/core/shuffle/RandomPartitioner.cc b/cpp/core/shuffle/RandomPartitioner.cc new file mode 100644 index 000000000000..06d87be40f7f --- /dev/null +++ b/cpp/core/shuffle/RandomPartitioner.cc @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "shuffle/RandomPartitioner.h" + +namespace gluten { + +arrow::Status gluten::RandomPartitioner::compute( + const int32_t* pidArr, + const int64_t numRows, + std::vector& row2Partition, + std::vector& partition2RowCount) { + std::fill(std::begin(partition2RowCount), std::end(partition2RowCount), 0); + row2Partition.resize(numRows); + + for (int32_t i = 0; i < numRows; ++i) { + row2Partition[i] = dist_(rng_); + } + + for (auto& pid : row2Partition) { + partition2RowCount[pid]++; + } + + return arrow::Status::OK(); +} + +arrow::Status gluten::RandomPartitioner::compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) { + auto index = static_cast(vectorIndex) << 32; + for (int32_t i = 0; i < numRows; ++i) { + int64_t combined = index | (i & 0xFFFFFFFFLL); + auto& vec = rowVectorIndexMap[dist_(rng_)]; + vec.push_back(combined); + } + + return arrow::Status::OK(); +} + +} // namespace gluten diff --git a/cpp/core/shuffle/RandomPartitioner.h b/cpp/core/shuffle/RandomPartitioner.h new file mode 100644 index 000000000000..77d00716943c --- /dev/null +++ b/cpp/core/shuffle/RandomPartitioner.h @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include "shuffle/Partitioner.h" + +namespace gluten { +class RandomPartitioner final : public Partitioner { + public: + RandomPartitioner(int32_t numPartitions) : Partitioner(numPartitions, false) { + std::random_device dev; + rng_.seed(dev()); + dist_ = std::uniform_int_distribution(0, numPartitions - 1); + } + + arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + std::vector& row2Partition, + std::vector& partition2RowCount) override; + + arrow::Status compute( + const int32_t* pidArr, + const int64_t numRows, + const int32_t vectorIndex, + std::unordered_map>& rowVectorIndexMap) override; + + private: + std::mt19937 rng_; + std::uniform_int_distribution dist_; +}; +} // namespace gluten diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 6d66ea506a7e..4eed625628f3 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -513,6 +513,7 @@ set(VELOX_SRCS operators/serializer/VeloxRowToColumnarConverter.cc operators/writer/VeloxParquetDatasource.cc shuffle/VeloxShuffleReader.cc + shuffle/VeloxShuffleWriter.cc shuffle/VeloxHashBasedShuffleWriter.cc shuffle/VeloxSortBasedShuffleWriter.cc substrait/SubstraitParser.cc diff --git a/cpp/velox/benchmarks/CMakeLists.txt b/cpp/velox/benchmarks/CMakeLists.txt index 903ec0d65825..1aa199b13696 100644 --- a/cpp/velox/benchmarks/CMakeLists.txt +++ b/cpp/velox/benchmarks/CMakeLists.txt @@ -39,8 +39,6 @@ add_velox_benchmark(parquet_write_benchmark ParquetWriteBenchmark.cc) add_velox_benchmark(plan_validator_util PlanValidatorUtil.cc) -add_velox_benchmark(shuffle_split_benchmark ShuffleSplitBenchmark.cc) - if(ENABLE_ORC) add_velox_benchmark(orc_converter exec/OrcConverter.cc) endif() diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc index b7a50800e4ea..d8c8c0c24a94 100644 --- a/cpp/velox/benchmarks/GenericBenchmark.cc +++ b/cpp/velox/benchmarks/GenericBenchmark.cc @@ -31,10 +31,10 @@ #include "compute/VeloxRuntime.h" #include "config/GlutenConfig.h" #include "shuffle/LocalPartitionWriter.h" -#include "shuffle/VeloxHashBasedShuffleWriter.h" #include "shuffle/VeloxShuffleWriter.h" #include "shuffle/rss/RssPartitionWriter.h" #include "utils/StringUtil.h" +#include "utils/Timer.h" #include "utils/VeloxArrowUtils.h" #include "utils/exception.h" #include "utils/tests/LocalRssClient.h" @@ -47,13 +47,18 @@ namespace { DEFINE_bool(print_result, true, "Print result for execution"); DEFINE_string(save_output, "", "Path to parquet file for saving the task output iterator"); DEFINE_bool(with_shuffle, false, "Add shuffle split at end."); -DEFINE_string(partitioning, "rr", "Short partitioning name. Valid options are rr, hash, range, single"); +DEFINE_string( + partitioning, + "rr", + "Short partitioning name. Valid options are rr, hash, range, single, random (only for test purpose)"); +DEFINE_string(shuffle_writer, "hash", "Shuffle writer type. Can be hash or sort"); DEFINE_bool(rss, false, "Mocking rss."); -DEFINE_bool(zstd, false, "Use ZSTD as shuffle compression codec"); -DEFINE_bool(qat_gzip, false, "Use QAT GZIP as shuffle compression codec"); -DEFINE_bool(qat_zstd, false, "Use QAT ZSTD as shuffle compression codec"); -DEFINE_bool(iaa_gzip, false, "Use IAA GZIP as shuffle compression codec"); +DEFINE_string( + compression, + "lz4", + "Specify the compression codec. Valid options are lz4, zstd, qat_gzip, qat_zstd, iaa_gzip"); DEFINE_int32(shuffle_partitions, 200, "Number of shuffle split (reducer) partitions"); +DEFINE_bool(run_shuffle, false, "Only run shuffle write."); DEFINE_bool(run_example, false, "Run the example and exit."); DEFINE_string(plan, "", "Path to input json file of the substrait plan."); @@ -61,33 +66,61 @@ DEFINE_string( split, "", "Path to input json file of the splits. Only valid for simulating the first stage. Use comma-separated list for multiple splits."); -DEFINE_string(data, "", "Path to input data files in parquet format, used for shuffle read."); +DEFINE_string(data, "", "Path to input data files in parquet format. Use comma-separated list for multiple files."); DEFINE_string(conf, "", "Path to the configuration file."); DEFINE_string(write_path, "/tmp", "Path to save the output from write tasks."); DEFINE_int64(memory_limit, std::numeric_limits::max(), "Memory limit used to trigger spill."); +DEFINE_string( + scan_mode, + "stream", + "Scan mode for reading parquet data." + "'stream' mode: Input file scan happens inside of the pipeline." + "'buffered' mode: First read all data into memory and feed the pipeline with it."); struct WriterMetrics { int64_t splitTime; int64_t evictTime; int64_t writeTime; int64_t compressTime; + + public: + explicit WriterMetrics() : splitTime(0), evictTime(0), writeTime(0), compressTime(0) {} }; +void setUpBenchmark(::benchmark::internal::Benchmark* bm) { + if (FLAGS_threads > 0) { + bm->Threads(FLAGS_threads); + } else { + bm->ThreadRange(1, std::thread::hardware_concurrency()); + } + if (FLAGS_iterations > 0) { + bm->Iterations(FLAGS_iterations); + } +} + std::shared_ptr createShuffleWriter( + Runtime* runtime, VeloxMemoryManager* memoryManager, const std::string& dataFile, const std::vector& localDirs) { PartitionWriterOptions partitionWriterOptions{}; - if (FLAGS_zstd) { + + // Configure compression. + if (FLAGS_compression == "lz4") { + partitionWriterOptions.codecBackend = CodecBackend::NONE; + partitionWriterOptions.compressionType = arrow::Compression::LZ4_FRAME; + partitionWriterOptions.compressionTypeStr = "lz4"; + } else if (FLAGS_compression == "zstd") { partitionWriterOptions.codecBackend = CodecBackend::NONE; partitionWriterOptions.compressionType = arrow::Compression::ZSTD; - } else if (FLAGS_qat_gzip) { + partitionWriterOptions.compressionTypeStr = "zstd"; + } else if (FLAGS_compression == "qat_gzip") { partitionWriterOptions.codecBackend = CodecBackend::QAT; partitionWriterOptions.compressionType = arrow::Compression::GZIP; - } else if (FLAGS_qat_zstd) { + } else if (FLAGS_compression == "qat_zstd") { partitionWriterOptions.codecBackend = CodecBackend::QAT; partitionWriterOptions.compressionType = arrow::Compression::ZSTD; - } else if (FLAGS_iaa_gzip) { + } else if (FLAGS_compression == "iaa_gzip") { partitionWriterOptions.codecBackend = CodecBackend::IAA; partitionWriterOptions.compressionType = arrow::Compression::GZIP; } @@ -111,28 +144,84 @@ std::shared_ptr createShuffleWriter( auto options = ShuffleWriterOptions{}; options.partitioning = gluten::toPartitioning(FLAGS_partitioning); - GLUTEN_ASSIGN_OR_THROW( - auto shuffleWriter, - VeloxHashBasedShuffleWriter::create( - FLAGS_shuffle_partitions, - std::move(partitionWriter), - std::move(options), - memoryManager->getLeafMemoryPool(), - memoryManager->getArrowMemoryPool())); - - return shuffleWriter; + if (FLAGS_shuffle_writer == "sort") { + options.shuffleWriterType = gluten::kSortShuffle; + } + auto shuffleWriter = runtime->createShuffleWriter( + FLAGS_shuffle_partitions, std::move(partitionWriter), std::move(options), memoryManager); + + return std::reinterpret_pointer_cast(shuffleWriter); } void populateWriterMetrics( const std::shared_ptr& shuffleWriter, - int64_t shuffleWriteTime, + int64_t totalTime, WriterMetrics& metrics) { metrics.compressTime += shuffleWriter->totalCompressTime(); metrics.evictTime += shuffleWriter->totalEvictTime(); metrics.writeTime += shuffleWriter->totalWriteTime(); - metrics.evictTime += - (shuffleWriteTime - shuffleWriter->totalCompressTime() - shuffleWriter->totalEvictTime() - - shuffleWriter->totalWriteTime()); + auto splitTime = totalTime - metrics.compressTime - metrics.evictTime - metrics.writeTime; + if (splitTime > 0) { + metrics.splitTime += splitTime; + } +} + +void setCpu(::benchmark::State& state) { + // Pin each threads to different CPU# starting from 0 or --cpu. + auto cpu = state.thread_index(); + if (FLAGS_cpu != -1) { + cpu += FLAGS_cpu; + } + LOG(INFO) << "Setting CPU for thread " << state.thread_index() << " to " << cpu; + gluten::setCpu(cpu); +} + +void runShuffle( + Runtime* runtime, + VeloxMemoryManager* memoryManager, + BenchmarkAllocationListener* listener, + const std::shared_ptr& resultIter, + WriterMetrics& metrics) { + std::string dataFile; + std::vector localDirs; + bool isFromEnv; + GLUTEN_THROW_NOT_OK(setLocalDirsAndDataFileFromEnv(dataFile, localDirs, isFromEnv)); + + auto shuffleWriter = createShuffleWriter(runtime, memoryManager, dataFile, localDirs); + listener->setShuffleWriter(shuffleWriter.get()); + + int64_t totalTime = 0; + { + gluten::ScopedTimer timer(&totalTime); + while (resultIter->hasNext()) { + GLUTEN_THROW_NOT_OK(shuffleWriter->write(resultIter->next(), ShuffleWriter::kMinMemLimit)); + } + GLUTEN_THROW_NOT_OK(shuffleWriter->stop()); + } + + populateWriterMetrics(shuffleWriter, totalTime, metrics); + // Cleanup shuffle outputs + cleanupShuffleOutput(dataFile, localDirs, isFromEnv); +} + +void updateBenchmarkMetrics( + ::benchmark::State& state, + const int64_t& elapsedTime, + const int64_t& readInputTime, + const WriterMetrics& writerMetrics) { + state.counters["read_input_time"] = + benchmark::Counter(readInputTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); + state.counters["elapsed_time"] = + benchmark::Counter(elapsedTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); + + state.counters["shuffle_write_time"] = benchmark::Counter( + writerMetrics.writeTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); + state.counters["shuffle_spill_time"] = benchmark::Counter( + writerMetrics.evictTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); + state.counters["shuffle_split_time"] = benchmark::Counter( + writerMetrics.splitTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); + state.counters["shuffle_compress_time"] = benchmark::Counter( + writerMetrics.compressTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); } } // namespace @@ -141,148 +230,140 @@ auto BM_Generic = [](::benchmark::State& state, const std::string& planFile, const std::vector& splitFiles, const std::vector& dataFiles, - const std::unordered_map& conf, + Runtime* runtime, FileReaderType readerType) { - // Pin each threads to different CPU# starting from 0 or --cpu. - if (FLAGS_cpu != -1) { - setCpu(FLAGS_cpu + state.thread_index()); - } else { - setCpu(state.thread_index()); - } - memory::MemoryManager::testingSetInstance({}); + setCpu(state); + auto listener = std::make_unique(FLAGS_memory_limit); + auto* listenerPtr = listener.get(); auto memoryManager = std::make_unique( - "generic_benchmark", - gluten::defaultMemoryAllocator(), - std::make_unique(FLAGS_memory_limit)); - auto runtime = Runtime::create(kVeloxRuntimeKind, conf); + "generic_benchmark", gluten::defaultMemoryAllocator(), std::move(listener)); + auto plan = getPlanFromFile("Plan", planFile); std::vector splits{}; for (const auto& splitFile : splitFiles) { splits.push_back(getPlanFromFile("ReadRel.LocalFiles", splitFile)); } - auto startTime = std::chrono::steady_clock::now(); - int64_t collectBatchTime = 0; - WriterMetrics writerMetrics{}; - for (auto _ : state) { - std::vector> inputIters; - std::vector inputItersRaw; - if (!dataFiles.empty()) { - for (const auto& input : dataFiles) { - inputIters.push_back(getInputIteratorFromFileReader(input, readerType)); - } - std::transform( - inputIters.begin(), - inputIters.end(), - std::back_inserter(inputItersRaw), - [](std::shared_ptr iter) { - return static_cast(iter->getInputIter()); - }); - } - runtime->injectWriteFilesTempPath(FLAGS_write_path); - runtime->parsePlan(reinterpret_cast(plan.data()), plan.size(), std::nullopt); - for (auto& split : splits) { - runtime->parseSplitInfo(reinterpret_cast(split.data()), split.size(), std::nullopt); - } - auto resultIter = - runtime->createResultIterator(memoryManager.get(), "/tmp/test-spill", std::move(inputIters), conf); - if (auto listener = dynamic_cast(memoryManager->getListener())) { - listener->setIterator(resultIter.get()); - } - auto veloxPlan = dynamic_cast(runtime)->getVeloxPlan(); - if (FLAGS_with_shuffle) { - int64_t shuffleWriteTime; - TIME_NANO_START(shuffleWriteTime); - std::string dataFile; - std::vector localDirs; - bool isFromEnv; - GLUTEN_THROW_NOT_OK(setLocalDirsAndDataFileFromEnv(dataFile, localDirs, isFromEnv)); - const auto& shuffleWriter = createShuffleWriter(memoryManager.get(), dataFile, localDirs); - while (resultIter->hasNext()) { - GLUTEN_THROW_NOT_OK(shuffleWriter->write(resultIter->next(), ShuffleWriter::kMinMemLimit)); + WriterMetrics writerMetrics{}; + int64_t readInputTime = 0; + int64_t elapsedTime = 0; + + { + ScopedTimer timer(&elapsedTime); + for (auto _ : state) { + std::vector> inputIters; + std::vector inputItersRaw; + if (!dataFiles.empty()) { + for (const auto& input : dataFiles) { + inputIters.push_back(getInputIteratorFromFileReader(input, readerType)); + } + std::transform( + inputIters.begin(), + inputIters.end(), + std::back_inserter(inputItersRaw), + [](std::shared_ptr iter) { + return static_cast(iter->getInputIter()); + }); } - GLUTEN_THROW_NOT_OK(shuffleWriter->stop()); - TIME_NANO_END(shuffleWriteTime); - populateWriterMetrics(shuffleWriter, shuffleWriteTime, writerMetrics); - // Cleanup shuffle outputs - cleanupShuffleOutput(dataFile, localDirs, isFromEnv); - } else { - // May write the output into file. - ArrowSchema cSchema; - toArrowSchema(veloxPlan->outputType(), memoryManager->getLeafMemoryPool().get(), &cSchema); - GLUTEN_ASSIGN_OR_THROW(auto outputSchema, arrow::ImportSchema(&cSchema)); - ArrowWriter writer{FLAGS_save_output}; - state.PauseTiming(); - if (!FLAGS_save_output.empty()) { - GLUTEN_THROW_NOT_OK(writer.initWriter(*(outputSchema.get()))); + runtime->injectWriteFilesTempPath(FLAGS_write_path); + runtime->parsePlan(reinterpret_cast(plan.data()), plan.size(), std::nullopt); + for (auto& split : splits) { + runtime->parseSplitInfo(reinterpret_cast(split.data()), split.size(), std::nullopt); } - state.ResumeTiming(); + auto resultIter = runtime->createResultIterator( + memoryManager.get(), "/tmp/test-spill", std::move(inputIters), runtime->getConfMap()); + listenerPtr->setIterator(resultIter.get()); - while (resultIter->hasNext()) { - auto array = resultIter->next()->exportArrowArray(); + if (FLAGS_with_shuffle) { + runShuffle(runtime, memoryManager.get(), listenerPtr, resultIter, writerMetrics); + } else { + // May write the output into file. + auto veloxPlan = dynamic_cast(runtime)->getVeloxPlan(); + + ArrowSchema cSchema; + toArrowSchema(veloxPlan->outputType(), memoryManager->getLeafMemoryPool().get(), &cSchema); + GLUTEN_ASSIGN_OR_THROW(auto outputSchema, arrow::ImportSchema(&cSchema)); + ArrowWriter writer{FLAGS_save_output}; state.PauseTiming(); - auto maybeBatch = arrow::ImportRecordBatch(array.get(), outputSchema); - if (!maybeBatch.ok()) { - state.SkipWithError(maybeBatch.status().message().c_str()); - return; + if (!FLAGS_save_output.empty()) { + GLUTEN_THROW_NOT_OK(writer.initWriter(*(outputSchema.get()))); } - if (FLAGS_print_result) { - LOG(INFO) << maybeBatch.ValueOrDie()->ToString(); + state.ResumeTiming(); + + while (resultIter->hasNext()) { + auto array = resultIter->next()->exportArrowArray(); + state.PauseTiming(); + auto maybeBatch = arrow::ImportRecordBatch(array.get(), outputSchema); + if (!maybeBatch.ok()) { + state.SkipWithError(maybeBatch.status().message().c_str()); + return; + } + if (FLAGS_print_result) { + LOG(INFO) << maybeBatch.ValueOrDie()->ToString(); + } + if (!FLAGS_save_output.empty()) { + GLUTEN_THROW_NOT_OK(writer.writeInBatches(maybeBatch.ValueOrDie())); + } } + + state.PauseTiming(); if (!FLAGS_save_output.empty()) { - GLUTEN_THROW_NOT_OK(writer.writeInBatches(maybeBatch.ValueOrDie())); + GLUTEN_THROW_NOT_OK(writer.closeWriter()); } + state.ResumeTiming(); } - state.PauseTiming(); - if (!FLAGS_save_output.empty()) { - GLUTEN_THROW_NOT_OK(writer.closeWriter()); - } - state.ResumeTiming(); - } - - collectBatchTime += - std::accumulate(inputItersRaw.begin(), inputItersRaw.end(), 0, [](int64_t sum, FileReaderIterator* iter) { - return sum + iter->getCollectBatchTime(); - }); + readInputTime += + std::accumulate(inputItersRaw.begin(), inputItersRaw.end(), 0, [](int64_t sum, FileReaderIterator* iter) { + return sum + iter->getCollectBatchTime(); + }); - auto* rawIter = static_cast(resultIter->getInputIter()); - const auto* task = rawIter->task(); - const auto* planNode = rawIter->veloxPlan(); - auto statsStr = facebook::velox::exec::printPlanWithStats(*planNode, task->taskStats(), true); - LOG(INFO) << statsStr; + auto* rawIter = static_cast(resultIter->getInputIter()); + const auto* task = rawIter->task(); + const auto* planNode = rawIter->veloxPlan(); + auto statsStr = facebook::velox::exec::printPlanWithStats(*planNode, task->taskStats(), true); + LOG(INFO) << statsStr; + } } - Runtime::release(runtime); - auto endTime = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast(endTime - startTime).count(); + updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics); +}; - state.counters["collect_batch_time"] = - benchmark::Counter(collectBatchTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); - state.counters["elapsed_time"] = - benchmark::Counter(duration, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); - state.counters["shuffle_write_time"] = benchmark::Counter( - writerMetrics.writeTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); - state.counters["shuffle_spill_time"] = benchmark::Counter( - writerMetrics.evictTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); - state.counters["shuffle_split_time"] = benchmark::Counter( - writerMetrics.splitTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); - state.counters["shuffle_compress_time"] = benchmark::Counter( - writerMetrics.compressTime, benchmark::Counter::kAvgIterations, benchmark::Counter::OneK::kIs1000); +auto BM_ShuffleWrite = + [](::benchmark::State& state, const std::string& inputFile, Runtime* runtime, FileReaderType readerType) { + setCpu(state); + + auto listener = std::make_unique(FLAGS_memory_limit); + auto* listenerPtr = listener.get(); + auto memoryManager = std::make_unique( + "generic_benchmark", gluten::defaultMemoryAllocator(), std::move(listener)); + + WriterMetrics writerMetrics{}; + int64_t readInputTime = 0; + int64_t elapsedTime = 0; + { + ScopedTimer timer(&elapsedTime); + for (auto _ : state) { + auto resultIter = getInputIteratorFromFileReader(inputFile, readerType); + runShuffle(runtime, memoryManager.get(), listenerPtr, resultIter, writerMetrics); + + auto reader = static_cast(resultIter->getInputIter()); + readInputTime += reader->getCollectBatchTime(); + } + } - gluten::VeloxBackend::get()->tearDown(); -}; + updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics); + }; int main(int argc, char** argv) { ::benchmark::Initialize(&argc, argv); gflags::ParseCommandLineFlags(&argc, &argv, true); // Init Velox backend. - std::unordered_map backendConf; - std::unordered_map sessionConf; + auto backendConf = gluten::defaultConf(); + auto sessionConf = gluten::defaultConf(); backendConf.insert({gluten::kSparkBatchSize, std::to_string(FLAGS_batch_size)}); - backendConf.insert({kDebugModeEnabled, "true"}); if (!FLAGS_conf.empty()) { abortIfFileNotExists(FLAGS_conf); std::ifstream file(FLAGS_conf); @@ -334,6 +415,7 @@ int main(int argc, char** argv) { } initVeloxBackend(backendConf); + memory::MemoryManager::testingSetInstance({}); // Parse substrait plan, split file and data files. std::string substraitJsonFile = FLAGS_plan; @@ -352,6 +434,28 @@ int main(int argc, char** argv) { ::benchmark::Shutdown(); std::exit(EXIT_FAILURE); } + } else if (FLAGS_run_shuffle) { + std::string errorMsg{}; + if (FLAGS_data.empty()) { + errorMsg = "Missing '--split' or '--data' option."; + } else if (FLAGS_partitioning != "rr" && FLAGS_partitioning != "random") { + errorMsg = "--run-shuffle only support round-robin partitioning and random partitioning."; + } + if (errorMsg.empty()) { + try { + dataFiles = gluten::splitPaths(FLAGS_data, true); + if (dataFiles.size() > 1) { + errorMsg = "Only one data file is allowed for shuffle write."; + } + } catch (const std::exception& e) { + errorMsg = e.what(); + } + } + if (!errorMsg.empty()) { + LOG(ERROR) << "Incorrect usage: " << errorMsg << std::endl; + ::benchmark::Shutdown(); + std::exit(EXIT_FAILURE); + } } else { // Validate input args. std::string errorMsg{}; @@ -363,15 +467,17 @@ int main(int argc, char** argv) { errorMsg = "Missing '--split' or '--data' option."; } - try { - if (!FLAGS_data.empty()) { - dataFiles = gluten::splitPaths(FLAGS_data, true); - } - if (!FLAGS_split.empty()) { - splitFiles = gluten::splitPaths(FLAGS_split, true); + if (errorMsg.empty()) { + try { + if (!FLAGS_data.empty()) { + dataFiles = gluten::splitPaths(FLAGS_data, true); + } + if (!FLAGS_split.empty()) { + splitFiles = gluten::splitPaths(FLAGS_split, true); + } + } catch (const std::exception& e) { + errorMsg = e.what(); } - } catch (const std::exception& e) { - errorMsg = e.what(); } if (!errorMsg.empty()) { @@ -406,20 +512,23 @@ int main(int argc, char** argv) { } } -#define GENERIC_BENCHMARK(NAME, READER_TYPE) \ - do { \ - auto* bm = ::benchmark::RegisterBenchmark( \ - NAME, BM_Generic, substraitJsonFile, splitFiles, dataFiles, sessionConf, READER_TYPE) \ - ->MeasureProcessCPUTime() \ - ->UseRealTime(); \ - if (FLAGS_threads > 0) { \ - bm->Threads(FLAGS_threads); \ - } else { \ - bm->ThreadRange(1, std::thread::hardware_concurrency()); \ - } \ - if (FLAGS_iterations > 0) { \ - bm->Iterations(FLAGS_iterations); \ - } \ + auto runtime = Runtime::create(kVeloxRuntimeKind, sessionConf); + +#define GENERIC_BENCHMARK(READER_TYPE) \ + do { \ + auto* bm = ::benchmark::RegisterBenchmark( \ + "GenericBenchmark", BM_Generic, substraitJsonFile, splitFiles, dataFiles, runtime, READER_TYPE) \ + ->MeasureProcessCPUTime() \ + ->UseRealTime(); \ + setUpBenchmark(bm); \ + } while (0) + +#define SHUFFLE_WRITE_BENCHMARK(READER_TYPE) \ + do { \ + auto* bm = ::benchmark::RegisterBenchmark("ShuffleWrite", BM_ShuffleWrite, dataFiles[0], runtime, READER_TYPE) \ + ->MeasureProcessCPUTime() \ + ->UseRealTime(); \ + setUpBenchmark(bm); \ } while (0) LOG(INFO) << "Using options: "; @@ -432,14 +541,28 @@ int main(int argc, char** argv) { LOG(INFO) << "write_path: " << FLAGS_write_path; if (dataFiles.empty()) { - GENERIC_BENCHMARK("SkipInput", FileReaderType::kNone); + GENERIC_BENCHMARK(FileReaderType::kNone); } else { - GENERIC_BENCHMARK("InputFromBatchVector", FileReaderType::kBuffered); - GENERIC_BENCHMARK("InputFromBatchStream", FileReaderType::kStream); + FileReaderType readerType; + if (FLAGS_scan_mode == "buffered") { + readerType = FileReaderType::kBuffered; + LOG(INFO) << "Using buffered mode for reading parquet data."; + } else { + readerType = FileReaderType::kStream; + LOG(INFO) << "Using stream mode for reading parquet data."; + } + if (FLAGS_run_shuffle) { + SHUFFLE_WRITE_BENCHMARK(readerType); + } else { + GENERIC_BENCHMARK(readerType); + } } ::benchmark::RunSpecifiedBenchmarks(); ::benchmark::Shutdown(); + Runtime::release(runtime); + gluten::VeloxBackend::get()->tearDown(); + return 0; } diff --git a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc index 7e9959797390..894c35351f17 100644 --- a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc +++ b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc @@ -307,7 +307,7 @@ class GoogleBenchmarkVeloxParquetWriteCacheScanBenchmark : public GoogleBenchmar // GoogleBenchmarkArrowParquetWriteCacheScanBenchmark usage // ./parquet_write_benchmark --threads=1 --file /mnt/DP_disk1/int.parquet --output /tmp/parquet-write int main(int argc, char** argv) { - initVeloxBackend(); + gluten::initVeloxBackend(); uint32_t iterations = 1; uint32_t threads = 1; std::string datafile; diff --git a/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc b/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc deleted file mode 100644 index 4a4bb69b8d78..000000000000 --- a/cpp/velox/benchmarks/ShuffleSplitBenchmark.cc +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "benchmarks/common/BenchmarkUtils.h" -#include "memory/ColumnarBatch.h" -#include "shuffle/LocalPartitionWriter.h" -#include "shuffle/VeloxHashBasedShuffleWriter.h" -#include "shuffle/VeloxShuffleWriter.h" -#include "utils/TestUtils.h" -#include "utils/VeloxArrowUtils.h" -#include "utils/macros.h" - -void printTrace(void) { - char** strings; - size_t i, size; - enum Constexpr { kMaxSize = 1024 }; - void* array[kMaxSize]; - size = backtrace(array, kMaxSize); - strings = backtrace_symbols(array, size); - for (i = 0; i < size; i++) - printf(" %s\n", strings[i]); - puts(""); - free(strings); -} - -using arrow::RecordBatchReader; -using arrow::Status; - -using gluten::GlutenException; -using gluten::ShuffleWriterOptions; -using gluten::VeloxShuffleWriter; - -DEFINE_int32(partitions, -1, "Shuffle partitions"); -DEFINE_string(file, "", "Input file to split"); - -namespace gluten { - -const uint16_t kBatchBufferSize = 4096; -const uint16_t kPartitionBufferSize = 4096; - -class BenchmarkShuffleSplit { - public: - BenchmarkShuffleSplit(std::string fileName) { - getRecordBatchReader(fileName); - } - - void getRecordBatchReader(const std::string& inputFile) { - std::unique_ptr<::parquet::arrow::FileReader> parquetReader; - std::shared_ptr recordBatchReader; - - std::shared_ptr fs; - std::string fileName; - GLUTEN_ASSIGN_OR_THROW(fs, arrow::fs::FileSystemFromUriOrPath(inputFile, &fileName)) - - GLUTEN_ASSIGN_OR_THROW(file_, fs->OpenInputFile(fileName)); - - properties_.set_batch_size(kBatchBufferSize); - properties_.set_pre_buffer(false); - properties_.set_use_threads(false); - - GLUTEN_THROW_NOT_OK(::parquet::arrow::FileReader::Make( - arrow::default_memory_pool(), ::parquet::ParquetFileReader::Open(file_), properties_, &parquetReader)); - - GLUTEN_THROW_NOT_OK(parquetReader->GetSchema(&schema_)); - - auto numRowgroups = parquetReader->num_row_groups(); - - for (int i = 0; i < numRowgroups; ++i) { - rowGroupIndices_.push_back(i); - } - - auto numColumns = schema_->num_fields(); - for (int i = 0; i < numColumns; ++i) { - columnIndices_.push_back(i); - } - } - - void operator()(benchmark::State& state) { - if (FLAGS_cpu != -1) { - setCpu(FLAGS_cpu + state.thread_index()); - } else { - setCpu(state.thread_index()); - } - - auto options = ShuffleWriterOptions{}; - options.bufferSize = kPartitionBufferSize; - options.partitioning = Partitioning::kRoundRobin; - std::string dataFile; - std::vector localDirs; - bool isFromEnv; - GLUTEN_THROW_NOT_OK(setLocalDirsAndDataFileFromEnv(dataFile, localDirs, isFromEnv)); - - std::shared_ptr shuffleWriter; - int64_t elapseRead = 0; - int64_t numBatches = 0; - int64_t numRows = 0; - int64_t splitTime = 0; - auto startTime = std::chrono::steady_clock::now(); - - doSplit( - shuffleWriter, - elapseRead, - numBatches, - numRows, - splitTime, - FLAGS_partitions, - std::move(options), - dataFile, - localDirs, - state); - auto endTime = std::chrono::steady_clock::now(); - auto totalTime = (endTime - startTime).count(); - - cleanupShuffleOutput(dataFile, localDirs, isFromEnv); - - state.SetBytesProcessed(int64_t(shuffleWriter->rawPartitionBytes())); - - state.counters["rowgroups"] = - benchmark::Counter(rowGroupIndices_.size(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["columns"] = - benchmark::Counter(columnIndices_.size(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["batches"] = - benchmark::Counter(numBatches, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["num_rows"] = - benchmark::Counter(numRows, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["num_partitions"] = - benchmark::Counter(FLAGS_partitions, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["batch_buffer_size"] = - benchmark::Counter(kBatchBufferSize, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1024); - state.counters["split_buffer_size"] = - benchmark::Counter(kPartitionBufferSize, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1024); - - state.counters["bytes_spilled"] = benchmark::Counter( - shuffleWriter->totalBytesEvicted(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1024); - state.counters["bytes_written"] = benchmark::Counter( - shuffleWriter->totalBytesWritten(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1024); - state.counters["bytes_raw"] = benchmark::Counter( - shuffleWriter->rawPartitionBytes(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1024); - - state.counters["parquet_parse"] = - benchmark::Counter(elapseRead, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["write_time"] = benchmark::Counter( - shuffleWriter->totalWriteTime(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["spill_time"] = benchmark::Counter( - shuffleWriter->totalEvictTime(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - state.counters["compress_time"] = benchmark::Counter( - shuffleWriter->totalCompressTime(), benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - - splitTime = splitTime - shuffleWriter->totalEvictTime() - shuffleWriter->totalCompressTime() - - shuffleWriter->totalWriteTime(); - - state.counters["split_time"] = - benchmark::Counter(splitTime, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - - state.counters["total_time"] = - benchmark::Counter(totalTime, benchmark::Counter::kAvgThreads, benchmark::Counter::OneK::kIs1000); - shuffleWriter.reset(); - } - - protected: - long setCpu(uint32_t cpuindex) { - cpu_set_t cs; - CPU_ZERO(&cs); - CPU_SET(cpuindex, &cs); - return sched_setaffinity(0, sizeof(cs), &cs); - } - - virtual void doSplit( - std::shared_ptr& shuffleWriter, - int64_t& elapseRead, - int64_t& numBatches, - int64_t& numRows, - int64_t& splitTime, - const int numPartitions, - ShuffleWriterOptions options, - const std::string& dataFile, - const std::vector& localDirs, - benchmark::State& state) {} - - protected: - std::shared_ptr file_; - std::vector rowGroupIndices_; - std::vector columnIndices_; - std::shared_ptr schema_; - parquet::ArrowReaderProperties properties_; -}; - -class BenchmarkShuffleSplitCacheScanBenchmark : public BenchmarkShuffleSplit { - public: - BenchmarkShuffleSplitCacheScanBenchmark(std::string filename) : BenchmarkShuffleSplit(filename) {} - - protected: - void doSplit( - std::shared_ptr& shuffleWriter, - int64_t& elapseRead, - int64_t& numBatches, - int64_t& numRows, - int64_t& splitTime, - const int numPartitions, - ShuffleWriterOptions options, - const std::string& dataFile, - const std::vector& localDirs, - benchmark::State& state) { - std::vector localColumnIndices; - // local_column_indices.push_back(0); - /* local_column_indices.push_back(0); - local_column_indices.push_back(1); - local_column_indices.push_back(2); - local_column_indices.push_back(4); - local_column_indices.push_back(5); - local_column_indices.push_back(6); - local_column_indices.push_back(7); -*/ - localColumnIndices.push_back(8); - localColumnIndices.push_back(9); - localColumnIndices.push_back(13); - localColumnIndices.push_back(14); - localColumnIndices.push_back(15); - - std::shared_ptr localSchema; - arrow::FieldVector fields; - fields.push_back(schema_->field(8)); - fields.push_back(schema_->field(9)); - fields.push_back(schema_->field(13)); - fields.push_back(schema_->field(14)); - fields.push_back(schema_->field(15)); - localSchema = std::make_shared(fields); - - if (state.thread_index() == 0) - LOG(INFO) << localSchema->ToString(); - - auto partitionWriter = std::make_unique( - numPartitions, PartitionWriterOptions{}, defaultArrowMemoryPool().get(), dataFile, localDirs); - GLUTEN_ASSIGN_OR_THROW( - shuffleWriter, - VeloxHashBasedShuffleWriter::create( - numPartitions, - std::move(partitionWriter), - std::move(options), - defaultLeafVeloxMemoryPool(), - defaultArrowMemoryPool().get())); - - std::shared_ptr recordBatch; - - std::unique_ptr<::parquet::arrow::FileReader> parquetReader; - std::shared_ptr recordBatchReader; - GLUTEN_THROW_NOT_OK(::parquet::arrow::FileReader::Make( - defaultArrowMemoryPool().get(), ::parquet::ParquetFileReader::Open(file_), properties_, &parquetReader)); - - std::vector> batches; - GLUTEN_THROW_NOT_OK(parquetReader->GetRecordBatchReader(rowGroupIndices_, localColumnIndices, &recordBatchReader)); - do { - TIME_NANO_OR_THROW(elapseRead, recordBatchReader->ReadNext(&recordBatch)); - - if (recordBatch) { - batches.push_back(recordBatch); - numBatches += 1; - numRows += recordBatch->num_rows(); - } - } while (recordBatch); - LOG(INFO) << "parquet parse done elapsed time " << elapseRead / 1000000 << " ms "; - LOG(INFO) << "batches = " << numBatches << " rows = " << numRows; - - for (auto _ : state) { - for_each( - batches.cbegin(), - batches.cend(), - [&shuffleWriter, &splitTime](const std::shared_ptr& recordBatch) { - std::shared_ptr cb; - ARROW_ASSIGN_OR_THROW(cb, recordBatch2VeloxColumnarBatch(*recordBatch)); - TIME_NANO_OR_THROW(splitTime, shuffleWriter->write(cb, ShuffleWriter::kMinMemLimit)); - }); - // LOG(INFO) << " split done memory allocated = " << - // options.memoryPool->bytes_allocated(); - } - - TIME_NANO_OR_THROW(splitTime, shuffleWriter->stop()); - } -}; - -class BenchmarkShuffleSplitIterateScanBenchmark : public BenchmarkShuffleSplit { - public: - BenchmarkShuffleSplitIterateScanBenchmark(std::string filename) : BenchmarkShuffleSplit(filename) {} - - protected: - void doSplit( - std::shared_ptr& shuffleWriter, - int64_t& elapseRead, - int64_t& numBatches, - int64_t& numRows, - int64_t& splitTime, - const int numPartitions, - ShuffleWriterOptions options, - const std::string& dataFile, - const std::vector& localDirs, - benchmark::State& state) { - if (state.thread_index() == 0) - LOG(INFO) << schema_->ToString(); - - auto partitionWriter = std::make_unique( - numPartitions, PartitionWriterOptions{}, defaultArrowMemoryPool().get(), dataFile, localDirs); - GLUTEN_ASSIGN_OR_THROW( - shuffleWriter, - VeloxHashBasedShuffleWriter::create( - numPartitions, - std::move(partitionWriter), - std::move(options), - defaultLeafVeloxMemoryPool(), - defaultArrowMemoryPool().get())); - - std::shared_ptr recordBatch; - - std::unique_ptr<::parquet::arrow::FileReader> parquetReader; - std::shared_ptr recordBatchReader; - GLUTEN_THROW_NOT_OK(::parquet::arrow::FileReader::Make( - defaultArrowMemoryPool().get(), ::parquet::ParquetFileReader::Open(file_), properties_, &parquetReader)); - - for (auto _ : state) { - std::vector> batches; - GLUTEN_THROW_NOT_OK(parquetReader->GetRecordBatchReader(rowGroupIndices_, columnIndices_, &recordBatchReader)); - TIME_NANO_OR_THROW(elapseRead, recordBatchReader->ReadNext(&recordBatch)); - while (recordBatch) { - numBatches += 1; - numRows += recordBatch->num_rows(); - std::shared_ptr cb; - ARROW_ASSIGN_OR_THROW(cb, recordBatch2VeloxColumnarBatch(*recordBatch)); - TIME_NANO_OR_THROW(splitTime, shuffleWriter->write(cb, ShuffleWriter::kMinMemLimit)); - TIME_NANO_OR_THROW(elapseRead, recordBatchReader->ReadNext(&recordBatch)); - } - } - TIME_NANO_OR_THROW(splitTime, shuffleWriter->stop()); - } -}; - -} // namespace gluten - -int main(int argc, char** argv) { - benchmark::Initialize(&argc, argv); - gflags::ParseCommandLineFlags(&argc, &argv, true); - - if (FLAGS_file.size() == 0) { - LOG(WARNING) << "No input data file. Please specify via argument --file"; - } - - if (FLAGS_partitions == -1) { - FLAGS_partitions = std::thread::hardware_concurrency(); - } - - gluten::BenchmarkShuffleSplitIterateScanBenchmark iterateScanBenchmark(FLAGS_file); - - auto bm = benchmark::RegisterBenchmark("BenchmarkShuffleSplit::IterateScan", iterateScanBenchmark) - ->ReportAggregatesOnly(false) - ->MeasureProcessCPUTime() - ->Unit(benchmark::kSecond); - - if (FLAGS_threads > 0) { - bm->Threads(FLAGS_threads); - } else { - bm->ThreadRange(1, std::thread::hardware_concurrency()); - } - if (FLAGS_iterations > 0) { - bm->Iterations(FLAGS_iterations); - } - - benchmark::RunSpecifiedBenchmarks(); - benchmark::Shutdown(); -} diff --git a/cpp/velox/benchmarks/common/BenchmarkUtils.cc b/cpp/velox/benchmarks/common/BenchmarkUtils.cc index a9f6f0838cfa..c3baa2f33915 100644 --- a/cpp/velox/benchmarks/common/BenchmarkUtils.cc +++ b/cpp/velox/benchmarks/common/BenchmarkUtils.cc @@ -31,14 +31,18 @@ DEFINE_int32(cpu, -1, "Run benchmark on specific CPU"); DEFINE_int32(threads, 1, "The number of threads to run this benchmark"); DEFINE_int32(iterations, 1, "The number of iterations to run this benchmark"); +namespace gluten { namespace { +std::unordered_map bmConfMap = defaultConf(); +} -std::unordered_map bmConfMap = {{gluten::kSparkBatchSize, std::to_string(FLAGS_batch_size)}}; - -} // namespace +std::unordered_map defaultConf() { + return { + {gluten::kSparkBatchSize, std::to_string(FLAGS_batch_size)}, + }; +} void initVeloxBackend(std::unordered_map& conf) { - conf[gluten::kGlogSeverityLevel] = "0"; gluten::VeloxBackend::create(conf); } @@ -190,9 +194,18 @@ void BenchmarkAllocationListener::allocationChanged(int64_t diff) { velox::succinctBytes(diff), velox::succinctBytes(usedBytes_)); auto neededBytes = usedBytes_ + diff - limit_; - auto spilledBytes = iterator_->spillFixedSize(neededBytes); + int64_t spilledBytes = 0; + if (iterator_) { + spilledBytes += iterator_->spillFixedSize(neededBytes); + } + if (spilledBytes < neededBytes && shuffleWriter_) { + int64_t reclaimed = 0; + GLUTEN_THROW_NOT_OK(shuffleWriter_->reclaimFixedSize(neededBytes - spilledBytes, &reclaimed)); + spilledBytes += reclaimed; + } LOG(INFO) << fmt::format("spill finish, got {}.", velox::succinctBytes(spilledBytes)); } else { usedBytes_ += diff; } } +} // namespace gluten diff --git a/cpp/velox/benchmarks/common/BenchmarkUtils.h b/cpp/velox/benchmarks/common/BenchmarkUtils.h index ff5e675f74ce..181e56807bcd 100644 --- a/cpp/velox/benchmarks/common/BenchmarkUtils.h +++ b/cpp/velox/benchmarks/common/BenchmarkUtils.h @@ -32,6 +32,7 @@ #include "memory/VeloxColumnarBatch.h" #include "memory/VeloxMemoryManager.h" #include "shuffle/Options.h" +#include "shuffle/ShuffleWriter.h" #include "utils/VeloxArrowUtils.h" #include "utils/exception.h" #include "velox/common/memory/Memory.h" @@ -41,6 +42,10 @@ DECLARE_int32(cpu); DECLARE_int32(threads); DECLARE_int32(iterations); +namespace gluten { + +std::unordered_map defaultConf(); + /// Initialize the Velox backend with default value. void initVeloxBackend(); @@ -111,10 +116,16 @@ class BenchmarkAllocationListener final : public gluten::AllocationListener { iterator_ = iterator; } + void setShuffleWriter(gluten::ShuffleWriter* shuffleWriter) { + shuffleWriter_ = shuffleWriter; + } + void allocationChanged(int64_t diff) override; private: uint64_t usedBytes_{0L}; uint64_t limit_{0L}; - gluten::ResultIterator* iterator_; + gluten::ResultIterator* iterator_{nullptr}; + gluten::ShuffleWriter* shuffleWriter_{nullptr}; }; +} // namespace gluten diff --git a/cpp/velox/benchmarks/common/FileReaderIterator.h b/cpp/velox/benchmarks/common/FileReaderIterator.h index 3fa94b6afba5..16db58ce4569 100644 --- a/cpp/velox/benchmarks/common/FileReaderIterator.h +++ b/cpp/velox/benchmarks/common/FileReaderIterator.h @@ -38,8 +38,6 @@ class FileReaderIterator : public ColumnarBatchIterator { virtual ~FileReaderIterator() = default; - virtual void createReader() = 0; - virtual std::shared_ptr getSchema() = 0; int64_t getCollectBatchTime() const { diff --git a/cpp/velox/benchmarks/common/ParquetReaderIterator.h b/cpp/velox/benchmarks/common/ParquetReaderIterator.h index e654dc1897b2..6d162e4b68d5 100644 --- a/cpp/velox/benchmarks/common/ParquetReaderIterator.h +++ b/cpp/velox/benchmarks/common/ParquetReaderIterator.h @@ -27,7 +27,7 @@ class ParquetReaderIterator : public FileReaderIterator { public: explicit ParquetReaderIterator(const std::string& path) : FileReaderIterator(path) {} - void createReader() override { + void createReader() { parquet::ArrowReaderProperties properties = parquet::default_arrow_reader_properties(); properties.set_batch_size(FLAGS_batch_size); GLUTEN_THROW_NOT_OK(parquet::arrow::FileReader::Make( diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index 44f04ef31ae7..738ce99a3bc7 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -35,7 +35,9 @@ #include "utils/VeloxArrowUtils.h" #ifdef ENABLE_HDFS + #include "operators/writer/VeloxParquetDatasourceHDFS.h" + #endif #ifdef ENABLE_S3 @@ -189,17 +191,15 @@ std::shared_ptr VeloxRuntime::createShuffleWriter( auto ctxPool = getLeafVeloxPool(memoryManager); auto arrowPool = memoryManager->getArrowMemoryPool(); std::shared_ptr shuffleWriter; - if (options.shuffleWriterType == kHashShuffle) { - GLUTEN_ASSIGN_OR_THROW( - shuffleWriter, - VeloxHashBasedShuffleWriter::create( - numPartitions, std::move(partitionWriter), std::move(options), ctxPool, arrowPool)); - } else if (options.shuffleWriterType == kSortShuffle) { - GLUTEN_ASSIGN_OR_THROW( - shuffleWriter, - VeloxSortBasedShuffleWriter::create( - numPartitions, std::move(partitionWriter), std::move(options), ctxPool, arrowPool)); - } + GLUTEN_ASSIGN_OR_THROW( + shuffleWriter, + VeloxShuffleWriter::create( + options.shuffleWriterType, + numPartitions, + std::move(partitionWriter), + std::move(options), + ctxPool, + arrowPool)); return shuffleWriter; } diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.cc b/cpp/velox/shuffle/VeloxShuffleWriter.cc new file mode 100644 index 000000000000..4b4f73f9463c --- /dev/null +++ b/cpp/velox/shuffle/VeloxShuffleWriter.cc @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "shuffle/VeloxShuffleWriter.h" +#include "shuffle/VeloxHashBasedShuffleWriter.h" +#include "shuffle/VeloxSortBasedShuffleWriter.h" + +namespace gluten { +arrow::Result> VeloxShuffleWriter::create( + ShuffleWriterType type, + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* arrowPool) { + std::shared_ptr shuffleWriter; + switch (type) { + case kHashShuffle: + return VeloxHashBasedShuffleWriter::create( + numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool); + case kSortShuffle: + return VeloxSortBasedShuffleWriter::create( + numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool); + default: + return arrow::Status::Invalid("Unsupported shuffle writer type: ", std::to_string(type)); + } +} +} // namespace gluten diff --git a/cpp/velox/shuffle/VeloxShuffleWriter.h b/cpp/velox/shuffle/VeloxShuffleWriter.h index 7318867fc590..0b49789c6d89 100644 --- a/cpp/velox/shuffle/VeloxShuffleWriter.h +++ b/cpp/velox/shuffle/VeloxShuffleWriter.h @@ -37,6 +37,7 @@ #include #include "memory/VeloxMemoryManager.h" +#include "shuffle/Options.h" #include "shuffle/PartitionWriter.h" #include "shuffle/Partitioner.h" #include "shuffle/ShuffleWriter.h" @@ -48,6 +49,14 @@ namespace gluten { class VeloxShuffleWriter : public ShuffleWriter { public: + static arrow::Result> create( + ShuffleWriterType type, + uint32_t numPartitions, + std::unique_ptr partitionWriter, + ShuffleWriterOptions options, + std::shared_ptr veloxPool, + arrow::MemoryPool* arrowPool); + facebook::velox::RowVectorPtr getStrippedRowVector(const facebook::velox::RowVector& rv) { // get new row type auto& rowType = rv.type()->asRow(); diff --git a/docs/developers/MicroBenchmarks.md b/docs/developers/MicroBenchmarks.md index 7fc2a535dcf1..21f222b42690 100644 --- a/docs/developers/MicroBenchmarks.md +++ b/docs/developers/MicroBenchmarks.md @@ -250,6 +250,93 @@ cd /path/to/gluten/cpp/build/velox/benchmarks --threads 1 --noprint-result --with-shuffle ``` +Developers can leverage the `--with-shuffle` option to benchmark the shuffle-write process by creating +a simple pipeline of `table scan + shuffle write` in Gluten. This can be done by dumping the micro benchmark +inputs from a first stage. The steps are demonstrated as below: + +1. Start spark-shell or pyspark + +We need to set `spark.gluten.sql.benchmark_task.stageId` and `spark.gluten.saveDir` to dump the inputs. +Normally, the stage id should be greater than 0. You can run the command in step 2 in advance to get the +right stage id in your case. We shall set `spark.default.parallelism` to 1 and `spark.sql.files.maxPartitionBytes` +large enough to make sure there will be only 1 task in the first stage. + +``` +# Start pyspark +./bin/pyspark --master local[*] \ +--conf spark.gluten.sql.benchmark_task.stageId=1 \ +--conf spark.gluten.saveDir=/path/to/saveDir \ +--conf spark.default.parallelism=1 \ +--conf spark.sql.files.maxPartitionBytes=10g +... # omit other spark & gluten config +``` + +2. Run the table-scan command to dump the plan for the first stage + +If simulating single or round-robin partitioning, the first stage can only have the table scan operator. + +``` +>>> spark.read.format("parquet").load("file:///example.parquet").show() +``` + +If simulating hash partitioning, there will be a projection for generating the hash partitioning key. +Therefore we need to explicitly run the `repartition` to generate the `scan + project` pipeline for the first stage. +Note that using different number of shuffle partitions here doesn't change the generated pipeline. + +``` +>>> spark.read.format("parquet").load("file:///example.parquet").repartition(10, "key1", "key2").show() +``` + +Simuating range partitioning is not supported. + +3. Run the micro benchmark with dumped inputs + +General configurations for shuffle write: + +- `--with-shuffle`: Add shuffle write process at the end of the pipeline +- `--shuffle-writer`: Specify shuffle writer type. Valid options are sort and hash. Default is hash. +- `--partitioning`: Specify partitioning type. Valid options are rr, hash and single. Defualt is rr. + The partitioning type should match the command in step 2. +- `--shuffle-partitions`: Specify number of shuffle partitions. +- `--compression`: By default, the compression codec for shuffle outputs is lz4. You can switch to other compression codecs + or use hardware accelerators Valid options are: lz4, zstd, qat-gzip, qat-zstd and iaa-gzip. The compression levels are fixed (use default compression level 1). + + Note using QAT or IAA codec requires Gluten cpp is built with these features. + Please check the corresponding section in [Velox document](../get-started/Velox.md) first for how to + setup, build and enable these features in Gluten. For QAT support, please + check [Intel® QuickAssist Technology (QAT) support](../get-started/Velox.md#intel-quickassist-technology-qat-support). + For IAA support, please + check [Intel® In-memory Analytics Accelerator (IAA/IAX) support](../get-started/Velox.md#intel-in-memory-analytics-accelerator-iaaiax-support) + +```shell +cd /path/to/gluten/cpp/build/velox/benchmarks +./generic_benchmark \ +--plan /path/to/saveDir/plan_1_0.json \ +--conf /path/to/saveDir/conf_1_0.ini \ +--split /path/to/saveDir/split_1_0_0.json \ +--with-shuffle \ +--shuffle-writer sort \ +--partitioning hash \ +--threads 1 +``` + +### Run shuffle write task only + +Developers can only run shuffle write task via specifying `--run-shuffle` and `--data` options. +The parquet format input will be read from arrow-parquet reader and sent to shuffle writer. +This option is similar to the `--with-shuffle` option, but it doesn't require the plan and split files. +The round-robin partitioner is used by default. Besides, random partitioning can be used for testing purpose. +By specifying option `--partitioning random`, the partitioner will generate a random partition id for each row. + +```shell +cd /path/to/gluten/cpp/build/velox/benchmarks +./generic_benchmark \ +--run-shuffle \ +--data /path/to/input_for_shuffle_write.parquet +--shuffle-writer sort \ +--threads 1 +``` + ## Simulate write tasks The last operator for a write task is a file write operator, and the output from Velox pipeline only @@ -265,20 +352,6 @@ cd /path/to/gluten/cpp/build/velox/benchmarks --write-path /absolute_path/ ``` -By default, the compression codec for shuffle outputs is LZ4. You can switch to other codecs by -adding one of the following argument flags to the command: - -- --zstd: ZSTD codec, compression level 1 -- --qat-gzip: QAT GZIP codec, compression level 1 -- --qat-zstd: QAT ZSTD codec, compression level 1 -- --iaa-gzip: IAA GZIP codec, compression level 1 - -Note using QAT or IAA codec requires Gluten cpp is built with these features. -Please check the corresponding section in [Velox document](../get-started/Velox.md) first for how to -setup, build and enable these features in Gluten. For QAT support, please -check [Intel® QuickAssist Technology (QAT) support](../get-started/Velox.md#intel-quickassist-technology-qat-support). -For IAA support, please -check [Intel® In-memory Analytics Accelerator (IAA/IAX) support](../get-started/Velox.md#intel-in-memory-analytics-accelerator-iaaiax-support) ## Simulate task spilling From fc04a630c9e291c8ea32ede81074ce3f6bb151e2 Mon Sep 17 00:00:00 2001 From: Wenzheng Liu Date: Mon, 17 Jun 2024 16:11:41 +0800 Subject: [PATCH 278/402] [GLUTEN-6110] Parallel run gluten ut and spark ut (#6090) 1. Add ut notes 2. move page index test to gluten package --- .../compatibility/GlutenFunctionSuite.scala | 4 ++-- .../parquet/GlutenParquetColumnIndexSuite.scala | 6 +++--- .../parquet/GlutenParquetFilterSuite.scala | 9 +++++---- .../sql => }/gluten/test/GlutenSQLTestUtils.scala | 2 +- .../sql => }/gluten/test/GlutenTPCBase.scala | 12 ++++++------ .../sql => }/gluten/test/GlutenTPCHBase.scala | 2 ++ docs/developers/NewToGluten.md | 14 ++++++++++++++ 7 files changed, 33 insertions(+), 16 deletions(-) rename backends-clickhouse/src/test/scala/org/apache/{spark/sql/gluten => gluten/execution}/compatibility/GlutenFunctionSuite.scala (96%) rename backends-clickhouse/src/test/scala/org/apache/{spark/sql/gluten => gluten/execution}/parquet/GlutenParquetColumnIndexSuite.scala (95%) rename backends-clickhouse/src/test/scala/org/apache/{spark/sql/gluten => gluten/execution}/parquet/GlutenParquetFilterSuite.scala (98%) rename backends-clickhouse/src/test/scala/org/apache/{spark/sql => }/gluten/test/GlutenSQLTestUtils.scala (96%) rename backends-clickhouse/src/test/scala/org/apache/{spark/sql => }/gluten/test/GlutenTPCBase.scala (80%) rename backends-clickhouse/src/test/scala/org/apache/{spark/sql => }/gluten/test/GlutenTPCHBase.scala (98%) diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenFunctionSuite.scala similarity index 96% rename from backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenFunctionSuite.scala index aaee8241206e..8a20558c4a8e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenFunctionSuite.scala @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.gluten.compatibility +package org.apache.gluten.execution.compatibility import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite +import org.apache.gluten.test.GlutenSQLTestUtils import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.internal.Logging -import org.apache.spark.sql.gluten.test.GlutenSQLTestUtils class GlutenFunctionSuite extends GlutenClickHouseWholeStageTransformerSuite diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetColumnIndexSuite.scala similarity index 95% rename from backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetColumnIndexSuite.scala index 05ed7ed6b842..0311594a18ad 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetColumnIndexSuite.scala @@ -14,15 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.gluten.parquet +package org.apache.gluten.execution.parquet import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseWholeStageTransformerSuite} +import org.apache.gluten.test.GlutenSQLTestUtils import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.gluten.test.GlutenSQLTestUtils import org.apache.spark.sql.internal.SQLConf case class ParquetData( @@ -98,6 +98,6 @@ class GlutenParquetColumnIndexSuite } override protected def sparkConf: SparkConf = super.sparkConf - .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false) + .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") .set("spark.gluten.sql.columnar.backend.ch.runtime_config.use_local_format", "true") } diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetFilterSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala similarity index 98% rename from backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetFilterSuite.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala index 1022b17e4311..a1b5801daddf 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetFilterSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala @@ -14,15 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.gluten.parquet +package org.apache.gluten.execution.parquet import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseWholeStageTransformerSuite} +import org.apache.gluten.test.GlutenSQLTestUtils import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.gluten.test.{GlutenSQLTestUtils, GlutenTPCHBase} +import org.apache.spark.sql.gluten.test.GlutenTPCHBase import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.Decimal @@ -45,8 +46,8 @@ class GlutenParquetFilterSuite override protected def sparkConf: SparkConf = super.sparkConf - .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false) - .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD, -1L) // disable broadcast + .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") + .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") // disable broadcast private val result: Array[Map[String, Seq[Predicate]]] = Array( Map( // q1 diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenSQLTestUtils.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala similarity index 96% rename from backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenSQLTestUtils.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala index 09ffb91ffb4e..9888baf9aac1 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenSQLTestUtils.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.gluten.test +package org.apache.gluten.test import org.apache.gluten.GlutenConfig diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCBase.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCBase.scala similarity index 80% rename from backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCBase.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCBase.scala index 21c6abb56fbe..224ad6443736 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCBase.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCBase.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.gluten.test +package org.apache.gluten.test import org.apache.spark.SparkConf import org.apache.spark.sql.internal.SQLConf @@ -27,12 +27,12 @@ trait GlutenTPCBase extends SharedSparkSession { override protected def sparkConf: SparkConf = { if (injectStats) { super.sparkConf - .set(SQLConf.MAX_TO_STRING_FIELDS, Int.MaxValue) - .set(SQLConf.CBO_ENABLED, true) - .set(SQLConf.PLAN_STATS_ENABLED, true) - .set(SQLConf.JOIN_REORDER_ENABLED, true) + .set(SQLConf.MAX_TO_STRING_FIELDS.key, s"${Int.MaxValue}") + .set(SQLConf.CBO_ENABLED.key, "true") + .set(SQLConf.PLAN_STATS_ENABLED.key, "true") + .set(SQLConf.JOIN_REORDER_ENABLED.key, "true") } else { - super.sparkConf.set(SQLConf.MAX_TO_STRING_FIELDS, Int.MaxValue) + super.sparkConf.set(SQLConf.MAX_TO_STRING_FIELDS.key, s"${Int.MaxValue}") } } diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCHBase.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCHBase.scala similarity index 98% rename from backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCHBase.scala rename to backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCHBase.scala index 4ed7e45131ce..685f185ac81f 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCHBase.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCHBase.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.gluten.test +import org.apache.gluten.test.GlutenTPCBase + import org.apache.spark.sql.catalyst.TableIdentifier trait GlutenTPCHBase extends GlutenTPCBase { diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 54f379148d39..a8862f7a5fa0 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -309,6 +309,20 @@ After the above installation, you can optionally do some configuration in Visual location, you might not need to change this setting. 3. Now, you can format your CMake files by right-clicking in a file and selecting `Format Document`. +### Add UT + +1. For Native Code Modifications: If you have modified native code, it is best to use gtest to test the native code. + A secondary option is to add Gluten UT to ensure coverage. + +2. For Gluten-Related Code Modifications: If you have modified code related to Gluten, it is preferable to add scalatest rather than JUnit. + Additionally, the test classes should be placed in the org.apache.gluten package. + +3. For Spark-Related Code Modifications: If you have modified code related to Spark, it is preferable to add scalatest rather than JUnit. + Additionally, the test classes should be placed in the org.apache.spark package. + +4. Placement of Non-Native Code UTs: Ensure that unit tests for non-native code are placed within org.apache.gluten and org.apache.spark packages. + This is important because the CI system runs unit tests from these two paths in parallel. Placing tests in other paths might cause your tests to be ignored. + # Debug cpp code with coredump ```bash From 150ccb92df132da33dc6f98fa5363403f546394a Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Mon, 17 Jun 2024 18:40:03 +0800 Subject: [PATCH 279/402] [CH] Support function base64/unbase64 (#6077) [CH] Support function base64/unbase64 Co-authored-by: liuneng1994 --- ...GlutenClickhouseStringFunctionsSuite.scala | 40 +++++++++++++++++++ .../expression/ExpressionMappings.scala | 2 + .../gluten/expression/ExpressionNames.scala | 2 + 3 files changed, 44 insertions(+) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala index 029a763c983c..163a8fedab7e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseStringFunctionsSuite.scala @@ -97,4 +97,44 @@ class GlutenClickhouseStringFunctionsSuite extends GlutenClickHouseWholeStageTra } } + test("base64") { + val tableName = "base64_table" + withTable(tableName) { + sql(s"create table $tableName(data String) using parquet") + sql(s""" + |insert into $tableName values + | ("hello") + """.stripMargin) + + val sql_str = + s""" + |select + | base64(data) + | from $tableName + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + + test("unbase64") { + val tableName = "unbase64_table" + withTable(tableName) { + sql(s"create table $tableName(data String) using parquet") + sql(s""" + |insert into $tableName values + | ("U3BhcmsgU1FM") + """.stripMargin) + + val sql_str = + s""" + |select + | unbase64(data) + | from $tableName + """.stripMargin + + runQueryAndCompare(sql_str) { _ => } + } + } + } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 230d91005e9c..f0082456fb18 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -101,6 +101,8 @@ object ExpressionMappings { Sig[Encode](ENCODE), Sig[Uuid](UUID), Sig[BitLength](BIT_LENGTH), + Sig[UnBase64](UNBASE64), + Sig[Base64](BASE64), // URL functions Sig[ParseUrl](PARSE_URL), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index f817612a1e8d..20db380180e3 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -127,6 +127,8 @@ object ExpressionNames { final val ENCODE = "encode" final val UUID = "uuid" final val BIT_LENGTH = "bit_length" + final val UNBASE64 = "unbase64" + final val BASE64 = "base64" // URL functions final val PARSE_URL = "parse_url" From 5b87efa5650529faad94206156dbd8964f22e65c Mon Sep 17 00:00:00 2001 From: Chang chen Date: Mon, 17 Jun 2024 21:39:46 +0800 Subject: [PATCH 280/402] [GLUTEN-6067][CH][Minor] Compile Spark-3.5 ut with backends-clickhouse (#6114) * Refactor GlutenV1WriteCommandSuite and GlutenInsertSuite, so we can compile spark-ut with spark-3.5 profile * fix warning --- gluten-ut/spark35/pom.xml | 44 +++++++++++++++++++ .../GlutenColumnarWriteTestSupport.scala | 26 +++++++++++ .../GlutenColumnarWriteTestSupport.scala | 27 ++++++++++++ .../GlutenV1WriteCommandSuite.scala | 12 ++--- .../spark/sql/sources/GlutenInsertSuite.scala | 42 +++++++++--------- 5 files changed, 125 insertions(+), 26 deletions(-) create mode 100644 gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/GlutenColumnarWriteTestSupport.scala create mode 100644 gluten-ut/spark35/src/test/backends-velox/org/apache/gluten/GlutenColumnarWriteTestSupport.scala diff --git a/gluten-ut/spark35/pom.xml b/gluten-ut/spark35/pom.xml index cf2129389a6e..2bf1c93a0052 100644 --- a/gluten-ut/spark35/pom.xml +++ b/gluten-ut/spark35/pom.xml @@ -63,6 +63,28 @@ test + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-sources + generate-sources + + add-test-source + + + + src/test/backends-clickhouse + + + + + + + backends-velox @@ -155,6 +177,28 @@ 2.19.0 + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-sources + generate-sources + + add-test-source + + + + src/test/backends-velox + + + + + + + diff --git a/gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/GlutenColumnarWriteTestSupport.scala b/gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/GlutenColumnarWriteTestSupport.scala new file mode 100644 index 000000000000..43b83afe9af3 --- /dev/null +++ b/gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/GlutenColumnarWriteTestSupport.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten + +import org.apache.spark.sql.execution.SparkPlan + +trait GlutenColumnarWriteTestSupport { + + def checkWriteFilesAndGetChild(sparkPlan: SparkPlan): SparkPlan = { + throw new UnsupportedOperationException("Clickhouse Backend does not support write files") + } +} diff --git a/gluten-ut/spark35/src/test/backends-velox/org/apache/gluten/GlutenColumnarWriteTestSupport.scala b/gluten-ut/spark35/src/test/backends-velox/org/apache/gluten/GlutenColumnarWriteTestSupport.scala new file mode 100644 index 000000000000..c7ad606bcf8d --- /dev/null +++ b/gluten-ut/spark35/src/test/backends-velox/org/apache/gluten/GlutenColumnarWriteTestSupport.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten + +import org.apache.spark.sql.execution.{SparkPlan, VeloxColumnarWriteFilesExec} + +trait GlutenColumnarWriteTestSupport { + + def checkWriteFilesAndGetChild(sparkPlan: SparkPlan): SparkPlan = { + assert(sparkPlan.isInstanceOf[VeloxColumnarWriteFilesExec]) + sparkPlan.asInstanceOf[VeloxColumnarWriteFilesExec].child + } +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenV1WriteCommandSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenV1WriteCommandSuite.scala index 3d277b94cc3e..fcaf75a4d5c1 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenV1WriteCommandSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenV1WriteCommandSuite.scala @@ -16,12 +16,13 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.GlutenColumnarWriteTestSupport import org.apache.gluten.execution.SortExecTransformer import org.apache.spark.sql.GlutenSQLTestsBaseTrait import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, NullsFirst, SortOrder} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Sort} -import org.apache.spark.sql.execution.{QueryExecution, SortExec, VeloxColumnarWriteFilesExec} +import org.apache.spark.sql.execution.{QueryExecution, SortExec} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, StringType} @@ -96,7 +97,8 @@ trait GlutenV1WriteCommandSuiteBase extends V1WriteCommandSuiteBase { class GlutenV1WriteCommandSuite extends V1WriteCommandSuite with GlutenV1WriteCommandSuiteBase - with GlutenSQLTestsBaseTrait { + with GlutenSQLTestsBaseTrait + with GlutenColumnarWriteTestSupport { testGluten( "SPARK-41914: v1 write with AQE and in-partition sorted - non-string partition column") { @@ -122,8 +124,7 @@ class GlutenV1WriteCommandSuite val executedPlan = FileFormatWriter.executedPlan.get val plan = if (enabled) { - assert(executedPlan.isInstanceOf[VeloxColumnarWriteFilesExec]) - executedPlan.asInstanceOf[VeloxColumnarWriteFilesExec].child + checkWriteFilesAndGetChild(executedPlan) } else { executedPlan.transformDown { case a: AdaptiveSparkPlanExec => a.executedPlan } } @@ -204,8 +205,7 @@ class GlutenV1WriteCommandSuite val executedPlan = FileFormatWriter.executedPlan.get val plan = if (enabled) { - assert(executedPlan.isInstanceOf[VeloxColumnarWriteFilesExec]) - executedPlan.asInstanceOf[VeloxColumnarWriteFilesExec].child + checkWriteFilesAndGetChild(executedPlan) } else { executedPlan.transformDown { case a: AdaptiveSparkPlanExec => a.executedPlan } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala index 2814c2e8cba6..084c2faa8c5c 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.sql.sources +import org.apache.gluten.GlutenColumnarWriteTestSupport import org.apache.gluten.execution.SortExecTransformer import org.apache.gluten.extension.GlutenPlan @@ -24,7 +25,7 @@ import org.apache.spark.executor.OutputMetrics import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.{CommandResultExec, QueryExecution, VeloxColumnarWriteFilesExec} +import org.apache.spark.sql.execution.{CommandResultExec, QueryExecution, SparkPlan} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.metric.SQLMetric @@ -38,7 +39,8 @@ import java.io.{File, IOException} class GlutenInsertSuite extends InsertSuite with GlutenSQLTestsBaseTrait - with AdaptiveSparkPlanHelper { + with AdaptiveSparkPlanHelper + with GlutenColumnarWriteTestSupport { override def sparkConf: SparkConf = { super.sparkConf.set("spark.sql.leafNodeDefaultParallelism", "1") @@ -60,13 +62,13 @@ class GlutenInsertSuite super.afterAll() } - private def checkAndGetWriteFiles(df: DataFrame): VeloxColumnarWriteFilesExec = { + private def checkWriteFilesAndGetChild(df: DataFrame): (SparkPlan, SparkPlan) = { val writeFiles = stripAQEPlan( df.queryExecution.executedPlan .asInstanceOf[CommandResultExec] .commandPhysicalPlan).children.head - assert(writeFiles.isInstanceOf[VeloxColumnarWriteFilesExec]) - writeFiles.asInstanceOf[VeloxColumnarWriteFilesExec] + val child = checkWriteFilesAndGetChild(writeFiles) + (writeFiles, child) } testGluten("insert partition table") { @@ -97,7 +99,7 @@ class GlutenInsertSuite val df = spark.sql("INSERT INTO TABLE pt partition(pt='a') SELECT * FROM VALUES(1, 'a'),(2, 'b')") spark.sparkContext.listenerBus.waitUntilEmpty() - checkAndGetWriteFiles(df) + checkWriteFilesAndGetChild(df) assert(taskMetrics.bytesWritten > 0) assert(taskMetrics.recordsWritten == 2) @@ -135,13 +137,13 @@ class GlutenInsertSuite private def validateDynamicPartitionWrite( df: DataFrame, expectedPartitionNames: Set[String]): Unit = { - val writeFiles = checkAndGetWriteFiles(df) + val (writeFiles, writeChild) = checkWriteFilesAndGetChild(df) assert( writeFiles .find(_.isInstanceOf[SortExecTransformer]) .isEmpty) // all operators should be transformed - assert(writeFiles.child.find(!_.isInstanceOf[GlutenPlan]).isEmpty) + assert(writeChild.find(!_.isInstanceOf[GlutenPlan]).isEmpty) val parts = spark.sessionState.catalog.listPartitionNames(TableIdentifier("pt")).toSet assert(parts == expectedPartitionNames) @@ -209,7 +211,7 @@ class GlutenInsertSuite spark.sql("CREATE TABLE t (c1 int, c2 string) USING PARQUET") val df = spark.sql("INSERT OVERWRITE TABLE t SELECT c1, c2 FROM source SORT BY c1") - val writeFiles = checkAndGetWriteFiles(df) + val (writeFiles, _) = checkWriteFilesAndGetChild(df) assert(writeFiles.find(x => x.isInstanceOf[SortExecTransformer]).isDefined) checkAnswer(spark.sql("SELECT * FROM t"), spark.sql("SELECT * FROM source SORT BY c1")) } @@ -244,7 +246,7 @@ class GlutenInsertSuite spark.sql("CREATE TABLE t1 USING PARQUET AS SELECT id as c1, id % 3 as c2 FROM range(10)") spark.sql("CREATE TABLE t2 (c1 long, c2 long) USING PARQUET") val df = spark.sql("INSERT INTO TABLE t2 SELECT c2, count(*) FROM t1 GROUP BY c2") - checkAndGetWriteFiles(df) + checkWriteFilesAndGetChild(df) } } @@ -257,7 +259,7 @@ class GlutenInsertSuite spark.sql("INSERT INTO TABLE t1 VALUES(1, 1),(2, 2)") spark.sql("CREATE TABLE t2 (c1 long, c2 long) USING PARQUET") val df = spark.sql("INSERT INTO TABLE t2 SELECT * FROM t1") - checkAndGetWriteFiles(df) + checkWriteFilesAndGetChild(df) } } @@ -405,7 +407,7 @@ class GlutenInsertSuite withTable("t") { sql(s"create table t(i boolean) using ${config.dataSource}") if (config.useDataFrames) { - Seq((false)).toDF.write.insertInto("t") + Seq(false).toDF.write.insertInto("t") } else { sql("insert into t select false") } @@ -420,12 +422,12 @@ class GlutenInsertSuite val incompatibleDefault = "Failed to execute ALTER TABLE ADD COLUMNS command because the destination " + "table column `s` has a DEFAULT value" - Seq(Config("parquet"), Config("parquet", true)).foreach { + Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach { config => withTable("t") { sql(s"create table t(i boolean) using ${config.dataSource}") if (config.useDataFrames) { - Seq((false)).toDF.write.insertInto("t") + Seq(false).toDF.write.insertInto("t") } else { sql("insert into t select false") } @@ -452,7 +454,7 @@ class GlutenInsertSuite withTable("t") { sql(s"create table t(i boolean) using ${config.dataSource}") if (config.useDataFrames) { - Seq((false)).toDF.write.insertInto("t") + Seq(false).toDF.write.insertInto("t") } else { sql("insert into t select false") } @@ -469,12 +471,12 @@ class GlutenInsertSuite val incompatibleDefault = "Failed to execute ALTER TABLE ADD COLUMNS command because the destination " + "table column `s` has a DEFAULT value" - Seq(Config("parquet"), Config("parquet", true)).foreach { + Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach { config => withTable("t") { sql(s"create table t(i boolean) using ${config.dataSource}") if (config.useDataFrames) { - Seq((false)).toDF.write.insertInto("t") + Seq(false).toDF.write.insertInto("t") } else { sql("insert into t select false") } @@ -501,7 +503,7 @@ class GlutenInsertSuite withTable("t") { sql(s"create table t(i boolean) using ${config.dataSource}") if (config.useDataFrames) { - Seq((false)).toDF.write.insertInto("t") + Seq(false).toDF.write.insertInto("t") } else { sql("insert into t select false") } @@ -566,12 +568,12 @@ class GlutenInsertSuite val incompatibleDefault = "Failed to execute ALTER TABLE ADD COLUMNS command because the destination " + "table column `s` has a DEFAULT value" - Seq(Config("parquet"), Config("parquet", true)).foreach { + Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach { config => withTable("t") { sql(s"create table t(i boolean) using ${config.dataSource}") if (config.useDataFrames) { - Seq((false)).toDF.write.insertInto("t") + Seq(false).toDF.write.insertInto("t") } else { sql("insert into t select false") } From e93596731a25fbe01afab0e88ce79b40ce645cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Tue, 18 Jun 2024 08:42:22 +0800 Subject: [PATCH 281/402] [VL] Doc update since Spark 3.5.1 has been fully supported (#6097) with the support native row index scan is merged, Spark 3.5.1 has been fully supported --- docs/get-started/Velox.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index 964744bd5eb5..d65b94fc1c26 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -9,7 +9,7 @@ parent: Getting-Started | Type | Version | |-------|---------------------------------| -| Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1(wip) | +| Spark | 3.2.2, 3.3.1, 3.4.2, 3.5.1 | | OS | Ubuntu20.04/22.04, Centos7/8 | | jdk | openjdk8/jdk17 | | scala | 2.12 | @@ -19,7 +19,7 @@ parent: Getting-Started Currently, Gluten+Velox backend is only tested on **Ubuntu20.04/Ubuntu22.04/Centos7/Centos8**. Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment. -Gluten only fully tested in CI with 3.2.2, 3.3.1 and 3.4.2. We will add/update supported/tested versions according to the upstream changes. +Gluten only fully tested in CI with 3.2.2, 3.3.1, 3.4.2 and 3.5.1. We will add/update supported/tested versions according to the upstream changes. We need to set up the `JAVA_HOME` env. Currently, Gluten supports **java 8** and **java 17**. From a722af30dab3c202033ce4f502d016de0a4edcda Mon Sep 17 00:00:00 2001 From: Jin Chengcheng Date: Tue, 18 Jun 2024 09:05:43 +0800 Subject: [PATCH 282/402] [VL] Fix RowToColumn metric convert time (#6106) --- .../execution/RowToVeloxColumnarExec.scala | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index be1bc64e21b8..5c9c5889bd13 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -145,10 +145,20 @@ object RowToVeloxColumnarExec { } } - def nativeConvert(row: UnsafeRow): ColumnarBatch = { + def convertToUnsafeRow(row: InternalRow): UnsafeRow = { + row match { + case unsafeRow: UnsafeRow => unsafeRow + case _ => + converter.apply(row) + } + } + + override def next(): ColumnarBatch = { + val firstRow = it.next() + val start = System.currentTimeMillis() + val row = convertToUnsafeRow(firstRow) var arrowBuf: ArrowBuf = null TaskResources.addRecycler("RowToColumnar_arrowBuf", 100) { - // Remind, remove isOpen here if (arrowBuf != null && arrowBuf.refCnt() != 0) { arrowBuf.close() } @@ -175,12 +185,14 @@ object RowToVeloxColumnarExec { rowLength += sizeInBytes.toLong rowCount += 1 + convertTime += System.currentTimeMillis() - start while (rowCount < columnBatchSize && !finished) { val iterHasNext = it.hasNext if (!iterHasNext) { finished = true } else { val row = it.next() + val start2 = System.currentTimeMillis() val unsafeRow = convertToUnsafeRow(row) val sizeInBytes = unsafeRow.getSizeInBytes if ((offset + sizeInBytes) > arrowBuf.capacity()) { @@ -198,36 +210,23 @@ object RowToVeloxColumnarExec { offset += sizeInBytes rowLength += sizeInBytes.toLong rowCount += 1 + convertTime += System.currentTimeMillis() - start2 } } numInputRows += rowCount + numOutputBatches += 1 + val startNative = System.currentTimeMillis() try { val handle = jniWrapper .nativeConvertRowToColumnar(r2cHandle, rowLength.toArray, arrowBuf.memoryAddress()) - ColumnarBatches.create(Runtimes.contextInstance(), handle) + val cb = ColumnarBatches.create(Runtimes.contextInstance(), handle) + convertTime += System.currentTimeMillis() - startNative + cb } finally { arrowBuf.close() arrowBuf = null } } - - def convertToUnsafeRow(row: InternalRow): UnsafeRow = { - row match { - case unsafeRow: UnsafeRow => unsafeRow - case _ => - converter.apply(row) - } - } - - override def next(): ColumnarBatch = { - val firstRow = it.next() - val start = System.currentTimeMillis() - val unsafeRow = convertToUnsafeRow(firstRow) - val cb = nativeConvert(unsafeRow) - numOutputBatches += 1 - convertTime += System.currentTimeMillis() - start - cb - } } Iterators .wrap(res) From eb653ba39bc7d79dcf86aea3a05375d74244753e Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 18 Jun 2024 09:16:33 +0800 Subject: [PATCH 283/402] [VL] RAS: New rule RemoveSort to remove unnecessary sorts (#6107) --- .../enumerated/EnumeratedTransform.scala | 1 + .../columnar/enumerated/RemoveSort.scala | 61 +++++++++++++++++++ .../org/apache/gluten/ras/path/Pattern.scala | 40 ++++++++++-- .../apache/gluten/ras/rule/PatternSuite.scala | 30 ++++++++- 4 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveSort.scala diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index 0b9dcc663246..9a54a101453f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -43,6 +43,7 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) private val rules = List( new PushFilterToScan(RasOffload.validator), + RemoveSort, RemoveFilter ) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveSort.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveSort.scala new file mode 100644 index 000000000000..5b5d5e541eb7 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveSort.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.enumerated + +import org.apache.gluten.execution.{HashAggregateExecBaseTransformer, ShuffledHashJoinExecTransformerBase, SortExecTransformer} +import org.apache.gluten.extension.GlutenPlan +import org.apache.gluten.ras.path.Pattern._ +import org.apache.gluten.ras.path.Pattern.Matchers._ +import org.apache.gluten.ras.rule.{RasRule, Shape} +import org.apache.gluten.ras.rule.Shapes._ + +import org.apache.spark.sql.catalyst.expressions.SortOrder +import org.apache.spark.sql.execution.SparkPlan + +/** + * Removes unnecessary sort if its parent doesn't require for sorted input. + * + * TODO: Sort's removal could be made much simpler once output ordering is added as a physical + * property in RAS planer. + */ +object RemoveSort extends RasRule[SparkPlan] { + private val appliedTypes: Seq[Class[_ <: GlutenPlan]] = + List(classOf[HashAggregateExecBaseTransformer], classOf[ShuffledHashJoinExecTransformerBase]) + + override def shift(node: SparkPlan): Iterable[SparkPlan] = { + assert(node.isInstanceOf[GlutenPlan]) + val newChildren = node.requiredChildOrdering.zip(node.children).map { + case (Nil, sort: SortExecTransformer) => + // Parent doesn't ask for sorted input from this child but a sort op was somehow added. + // Remove it. + sort.child + case (req, child) => + // Parent asks for sorted input from this child. Do nothing but an assertion. + assert(SortOrder.orderingSatisfies(child.outputOrdering, req)) + child + } + val out = List(node.withNewChildren(newChildren)) + out + } + override def shape(): Shape[SparkPlan] = pattern( + branch2[SparkPlan]( + or(appliedTypes.map(clazz[SparkPlan](_)): _*), + _ >= 1, + _ => node(clazz(classOf[GlutenPlan])) + ).build() + ) +} diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala index e60a94717654..f54b031b0aef 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/Pattern.scala @@ -87,14 +87,35 @@ object Pattern { override def children(count: Int): Seq[Node[T]] = (0 until count).map(_ => ignore[T]) } - private case class Branch[T <: AnyRef](matcher: Matcher[T], children: Seq[Node[T]]) + private case class Branch[T <: AnyRef](matcher: Matcher[T], children: Branch.ChildrenFactory[T]) extends Node[T] { override def skip(): Boolean = false - override def abort(node: CanonicalNode[T]): Boolean = node.childrenCount != children.size + override def abort(node: CanonicalNode[T]): Boolean = + !children.acceptsChildrenCount(node.childrenCount) override def matches(node: CanonicalNode[T]): Boolean = matcher(node.self()) override def children(count: Int): Seq[Node[T]] = { - assert(count == children.size) - children + assert(children.acceptsChildrenCount(count)) + (0 until count).map(children.child) + } + } + + private object Branch { + trait ChildrenFactory[T <: AnyRef] { + def child(index: Int): Node[T] + def acceptsChildrenCount(count: Int): Boolean + } + + object ChildrenFactory { + case class Plain[T <: AnyRef](nodes: Seq[Node[T]]) extends ChildrenFactory[T] { + override def child(index: Int): Node[T] = nodes(index) + override def acceptsChildrenCount(count: Int): Boolean = nodes.size == count + } + + case class Func[T <: AnyRef](arity: Int => Boolean, func: Int => Node[T]) + extends ChildrenFactory[T] { + override def child(index: Int): Node[T] = func(index) + override def acceptsChildrenCount(count: Int): Boolean = arity(count) + } } } @@ -102,8 +123,15 @@ object Pattern { def ignore[T <: AnyRef]: Node[T] = Ignore.INSTANCE.asInstanceOf[Node[T]] def node[T <: AnyRef](matcher: Matcher[T]): Node[T] = Single(matcher) def branch[T <: AnyRef](matcher: Matcher[T], children: Node[T]*): Node[T] = - Branch(matcher, children.toSeq) - def leaf[T <: AnyRef](matcher: Matcher[T]): Node[T] = Branch(matcher, List.empty) + Branch(matcher, Branch.ChildrenFactory.Plain(children.toSeq)) + // Similar to #branch, but with unknown arity. + def branch2[T <: AnyRef]( + matcher: Matcher[T], + arity: Int => Boolean, + children: Int => Node[T]): Node[T] = + Branch(matcher, Branch.ChildrenFactory.Func(arity, children)) + def leaf[T <: AnyRef](matcher: Matcher[T]): Node[T] = + Branch(matcher, Branch.ChildrenFactory.Plain(List.empty)) implicit class NodeImplicits[T <: AnyRef](node: Node[T]) { def build(): Pattern[T] = { diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala index 64b66bbaffae..dc7f5e883022 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/rule/PatternSuite.scala @@ -59,6 +59,29 @@ class PatternSuite extends AnyFunSuite { assert(pattern.matches(path, 1)) } + test("Match branch") { + val ras = + Ras[TestNode]( + PlanModelImpl, + CostModelImpl, + MetadataModelImpl, + PropertyModelImpl, + ExplainImpl, + RasRule.Factory.none()) + + val path1 = MockRasPath.mock(ras, Branch("n1", List())) + val path2 = MockRasPath.mock(ras, Branch("n1", List(Leaf("n2", 1)))) + val path3 = MockRasPath.mock(ras, Branch("n1", List(Leaf("n2", 1), Leaf("n3", 1)))) + + val pattern = + Pattern.branch2[TestNode](n => n.isInstanceOf[Branch], _ >= 1, _ => Pattern.any).build() + assert(!pattern.matches(path1, 1)) + assert(pattern.matches(path2, 1)) + assert(pattern.matches(path2, 2)) + assert(pattern.matches(path3, 1)) + assert(pattern.matches(path3, 2)) + } + test("Match unary") { val ras = Ras[TestNode]( @@ -231,17 +254,20 @@ object PatternSuite { case class Unary(name: String, child: TestNode) extends UnaryLike { override def selfCost(): Long = 1 - override def withNewChildren(child: TestNode): UnaryLike = copy(child = child) } case class Binary(name: String, left: TestNode, right: TestNode) extends BinaryLike { override def selfCost(): Long = 1 - override def withNewChildren(left: TestNode, right: TestNode): BinaryLike = copy(left = left, right = right) } + case class Branch(name: String, children: Seq[TestNode]) extends TestNode { + override def selfCost(): Long = 1 + override def withNewChildren(children: Seq[TestNode]): TestNode = copy(children = children) + } + case class DummyGroup() extends LeafLike { override def makeCopy(): LeafLike = throw new UnsupportedOperationException() override def selfCost(): Long = throw new UnsupportedOperationException() From 1fe5a779a4cb41c333cce8f202855d8c4e1e125c Mon Sep 17 00:00:00 2001 From: JiaKe Date: Tue, 18 Jun 2024 12:47:08 +0800 Subject: [PATCH 284/402] [VL] Daily Update Velox Version (2024_06_17) (#6109) --- cpp/velox/compute/WholeStageResultIterator.cc | 1 + ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 867d347cdc64..5920a08985f4 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -529,6 +529,7 @@ std::shared_ptr WholeStageResultIterator::createConnectorConfig() !veloxCfg_->get(kCaseSensitive, false) ? "true" : "false"; configs[velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "false"; configs[velox::connector::hive::HiveConfig::kParquetWriteTimestampUnitSession] = "6"; + configs[velox::connector::hive::HiveConfig::kReadTimestampUnitSession] = "6"; configs[velox::connector::hive::HiveConfig::kMaxPartitionsPerWritersSession] = std::to_string(veloxCfg_->get(kMaxPartitions, 10000)); configs[velox::connector::hive::HiveConfig::kIgnoreMissingFilesSession] = diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index d118ca0c9188..0e72469a43fc 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_14 +VELOX_BRANCH=2024_06_17 VELOX_HOME="" #Set on run gluten on HDFS From be0ada5c73a1002e350c9de2f0855abca9764373 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:45:58 +0800 Subject: [PATCH 285/402] [GLUTEN-6082][CH]Fix lag diff (#6085) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #6082) How was this patch tested? TEST BY UT --- .../backendsapi/clickhouse/CHBackend.scala | 21 ++++++++++++++++--- ...enClickHouseTPCHSaltNullParquetSuite.scala | 8 +++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index c79d0aaee800..1587b9ea3488 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -26,7 +26,7 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat._ import org.apache.spark.SparkEnv import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, DenseRank, Lag, Lead, NamedExpression, Rank, RowNumber} +import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, DenseRank, Expression, Lag, Lead, Literal, NamedExpression, Rank, RowNumber} import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning} import org.apache.spark.sql.execution.SparkPlan @@ -225,10 +225,25 @@ object CHBackendSettings extends BackendSettingsApi with Logging { func => { val aliasExpr = func.asInstanceOf[Alias] val wExpression = WindowFunctionsBuilder.extractWindowExpression(aliasExpr.child) + + def checkLagOrLead(third: Expression): Unit = { + third match { + case _: Literal => + allSupported = allSupported + case _ => + logInfo("Not support lag/lead function with default value not literal null") + allSupported = false + break + } + } + wExpression.windowFunction match { - case _: RowNumber | _: AggregateExpression | _: Rank | _: Lead | _: Lag | - _: DenseRank => + case _: RowNumber | _: AggregateExpression | _: Rank | _: DenseRank => allSupported = allSupported + case l: Lag => + checkLagOrLead(l.third) + case l: Lead => + checkLagOrLead(l.third) case _ => logDebug(s"Not support window function: ${wExpression.getClass}") allSupported = false diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 84f3901397ea..1d3bbec848bc 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -944,7 +944,15 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr |from nation |order by n_regionkey, n_nationkey, n_lag |""".stripMargin + val sql1 = + """ + | select n_regionkey, n_nationkey, + | lag(n_nationkey, 1, n_nationkey) OVER (PARTITION BY n_regionkey ORDER BY n_nationkey) as n_lag + |from nation + |order by n_regionkey, n_nationkey, n_lag + |""".stripMargin compareResultsAgainstVanillaSpark(sql, true, { _ => }) + compareResultsAgainstVanillaSpark(sql1, true, { _ => }, false) } test("window lag with null value") { From 1ed835848cc091e63e4137a92af08589d6b5b0de Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:47:09 +0800 Subject: [PATCH 286/402] [GLUTEN-6111][CH]Fix core problem of get_json_object #6113 What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #6111) How was this patch tested? test by ut --- .../resources/text-data/abnormal-json/data2.txt | Bin 36 -> 50 bytes .../Functions/SparkFunctionGetJsonObject.h | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/backends-clickhouse/src/test/resources/text-data/abnormal-json/data2.txt b/backends-clickhouse/src/test/resources/text-data/abnormal-json/data2.txt index 485fb6d7205a81a3a1079e46b72e008ade86b937..e323f5986f7bb930bd7b5c1b1ba278a8425edfd6 100644 GIT binary patch delta 19 acmY!}nxMj0Tcc!RU}&nXXJ%=v#03B@HUuRA delta 4 LcmXp~nV Date: Tue, 18 Jun 2024 13:50:57 +0800 Subject: [PATCH 287/402] [CI] Disable ENABLE_GWP_ASAN (#6119) What changes were proposed in this pull request? Even with #6092, we still get ci failed randomly https://opencicd.kyligence.com/job/gluten/job/gluten-ci/10132/ https://opencicd.kyligence.com/job/gluten/job/gluten-ci/10133/ Since we need export LD_PRELOAD in shell which casue comiler segment fault. img_v3_02bv_3404a655-a3f3-4cfa-9c79-1dce19f82c2g Let's disable ENABLE_GWP_ASAN first (Fixes: #6091) How was this patch tested? Existed UT --- cpp-ch/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp-ch/CMakeLists.txt b/cpp-ch/CMakeLists.txt index 1ef19b3b5651..82049bdd31bb 100644 --- a/cpp-ch/CMakeLists.txt +++ b/cpp-ch/CMakeLists.txt @@ -111,7 +111,7 @@ else() -DENABLE_DATASKETCHES=OFF -DENABLE_SQLITE=OFF -DENABLE_S2_GEOMETRY=OFF -DENABLE_ANNOY=OFF -DENABLE_ULID=OFF -DENABLE_MYSQL=OFF -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF -DENABLE_MSGPACK=OFF - -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF + -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF -DENABLE_GWP_ASAN=OFF -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build ${CH_BINARY_DIR} --target libch\" From 1e6d973b8408c003d32a96a859972b3a16d54090 Mon Sep 17 00:00:00 2001 From: Wenzheng Liu Date: Tue, 18 Jun 2024 14:30:13 +0800 Subject: [PATCH 288/402] [GLUTEN-6053][CH] Move collect native metrics from last hasNext to close and cancel (#6069) --- .../clickhouse/CHIteratorApi.scala | 128 ++++++++---------- .../GlutenClickHouseTPCHMetricsSuite.scala | 35 ++++- 2 files changed, 93 insertions(+), 70 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 5f13b96a3671..941237629569 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -25,10 +25,11 @@ import org.apache.gluten.substrait.plan.PlanNode import org.apache.gluten.substrait.rel._ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.utils.LogLevelUtil -import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator} +import org.apache.gluten.vectorized.{BatchIterator, CHNativeExpressionEvaluator, CloseableCHColumnBatchIterator, GeneralInIterator} import org.apache.spark.{InterruptibleIterator, SparkConf, TaskContext} import org.apache.spark.affinity.CHAffinity +import org.apache.spark.executor.InputMetrics import org.apache.spark.internal.Logging import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.datasources.FilePartition @@ -209,46 +210,26 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { val splitInfoByteArray = inputPartition .asInstanceOf[GlutenPartition] .splitInfosByteArray - val resIter = + val nativeIter = transKernel.createKernelWithBatchIterator( inputPartition.plan, splitInfoByteArray, inBatchIters, false) + val iter = new CollectMetricIterator( + nativeIter, + updateNativeMetrics, + updateInputMetrics, + context.taskMetrics().inputMetrics) + context.addTaskFailureListener( (ctx, _) => { if (ctx.isInterrupted()) { - resIter.cancel() + iter.cancel() } }) - context.addTaskCompletionListener[Unit](_ => resIter.close()) - val iter = new Iterator[Any] { - private val inputMetrics = context.taskMetrics().inputMetrics - private var outputRowCount = 0L - private var outputVectorCount = 0L - private var metricsUpdated = false - - override def hasNext: Boolean = { - val res = resIter.hasNext - // avoid to collect native metrics more than once, 'hasNext' is a idempotent operation - if (!res && !metricsUpdated) { - val nativeMetrics = resIter.getMetrics.asInstanceOf[NativeMetrics] - nativeMetrics.setFinalOutputMetrics(outputRowCount, outputVectorCount) - updateNativeMetrics(nativeMetrics) - updateInputMetrics(inputMetrics) - metricsUpdated = true - } - res - } - - override def next(): Any = { - val cb = resIter.next() - outputVectorCount += 1 - outputRowCount += cb.numRows() - cb - } - } + context.addTaskCompletionListener[Unit](_ => iter.close()) // TODO: SPARK-25083 remove the type erasure hack in data source scan new InterruptibleIterator( @@ -288,51 +269,16 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { materializeInput ) - val resIter = new Iterator[ColumnarBatch] { - private var outputRowCount = 0L - private var outputVectorCount = 0L - private var metricsUpdated = false - - override def hasNext: Boolean = { - val res = nativeIterator.hasNext - // avoid to collect native metrics more than once, 'hasNext' is a idempotent operation - if (!res && !metricsUpdated) { - val nativeMetrics = nativeIterator.getMetrics.asInstanceOf[NativeMetrics] - nativeMetrics.setFinalOutputMetrics(outputRowCount, outputVectorCount) - updateNativeMetrics(nativeMetrics) - metricsUpdated = true - } - res - } - - override def next(): ColumnarBatch = { - val cb = nativeIterator.next() - outputVectorCount += 1 - outputRowCount += cb.numRows() - cb - } - } - var closed = false - val cancelled = false - - def close(): Unit = { - closed = true - nativeIterator.close() - // relationHolder.clear() - } - - def cancel(): Unit = { - nativeIterator.cancel() - } + val iter = new CollectMetricIterator(nativeIterator, updateNativeMetrics, null, null) context.addTaskFailureListener( (ctx, _) => { if (ctx.isInterrupted()) { - cancel() + iter.cancel() } }) - context.addTaskCompletionListener[Unit](_ => close()) - new CloseableCHColumnBatchIterator(resIter, Some(pipelineTime)) + context.addTaskCompletionListener[Unit](_ => iter.close()) + new CloseableCHColumnBatchIterator(iter, Some(pipelineTime)) } } @@ -346,3 +292,47 @@ object CHIteratorApi { } } } + +class CollectMetricIterator( + val nativeIterator: BatchIterator, + val updateNativeMetrics: IMetrics => Unit, + val updateInputMetrics: InputMetricsWrapper => Unit, + val inputMetrics: InputMetrics +) extends Iterator[ColumnarBatch] { + private var outputRowCount = 0L + private var outputVectorCount = 0L + private var metricsUpdated = false + + override def hasNext: Boolean = { + nativeIterator.hasNext + } + + override def next(): ColumnarBatch = { + val cb = nativeIterator.next() + outputVectorCount += 1 + outputRowCount += cb.numRows() + cb + } + + def close(): Unit = { + collectStageMetrics() + nativeIterator.close() + } + + def cancel(): Unit = { + collectStageMetrics() + nativeIterator.cancel() + } + + private def collectStageMetrics(): Unit = { + if (!metricsUpdated) { + val nativeMetrics = nativeIterator.getMetrics.asInstanceOf[NativeMetrics] + nativeMetrics.setFinalOutputMetrics(outputRowCount, outputVectorCount) + updateNativeMetrics(nativeMetrics) + if (updateInputMetrics != null) { + updateInputMetrics(inputMetrics) + } + metricsUpdated = true + } + } +} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala index 1484d4653260..09fa3ff109f2 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.InputIteratorTransformer import scala.collection.JavaConverters._ class GlutenClickHouseTPCHMetricsSuite extends GlutenClickHouseTPCHAbstractSuite { - + private val parquetMaxBlockSize = 4096; override protected val needCopyParquetToTablePath = true override protected val tablesPath: String = basePath + "/tpch-data" @@ -38,6 +38,7 @@ class GlutenClickHouseTPCHMetricsSuite extends GlutenClickHouseTPCHAbstractSuite protected val metricsJsonFilePath: String = rootPath + "metrics-json" protected val substraitPlansDatPath: String = rootPath + "substrait-plans" + // scalastyle:off line.size.limit /** Run Gluten + ClickHouse Backend with SortShuffleManager */ override protected def sparkConf: SparkConf = { super.sparkConf @@ -45,10 +46,15 @@ class GlutenClickHouseTPCHMetricsSuite extends GlutenClickHouseTPCHAbstractSuite .set("spark.io.compression.codec", "LZ4") .set("spark.sql.shuffle.partitions", "1") .set("spark.sql.autoBroadcastJoinThreshold", "10MB") + .set("spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level", "DEBUG") + .set( + "spark.gluten.sql.columnar.backend.ch.runtime_settings.input_format_parquet_max_block_size", + s"$parquetMaxBlockSize") .set( "spark.gluten.sql.columnar.backend.ch.runtime_config.enable_streaming_aggregating", "true") } + // scalastyle:on line.size.limit override protected def createTPCHNotNullTables(): Unit = { createNotNullTPCHTablesInParquet(tablesPath) @@ -76,6 +82,33 @@ class GlutenClickHouseTPCHMetricsSuite extends GlutenClickHouseTPCHAbstractSuite } } + test("test simple limit query scan metrics") { + val sql = "select * from nation limit 5" + runSql(sql) { + df => + val plans = df.queryExecution.executedPlan.collect { + case scanExec: BasicScanExecTransformer => scanExec + } + assert(plans.size == 1) + assert(plans.head.metrics("numOutputRows").value === 25) + assert(plans.head.metrics("outputVectors").value === 1) + assert(plans.head.metrics("outputBytes").value > 0) + } + + val sql2 = "select * from lineitem limit 3" + runSql(sql2) { + df => + val plans = df.queryExecution.executedPlan.collect { + case scanExec: BasicScanExecTransformer => scanExec + } + assert(plans.size == 1) + // 1 block keep in SubstraitFileStep, and 4 blocks keep in other steps + assert(plans.head.metrics("numOutputRows").value === 5 * parquetMaxBlockSize) + assert(plans.head.metrics("outputVectors").value === 1) + assert(plans.head.metrics("outputBytes").value > 0) + } + } + test("test Generate metrics") { val sql = """ From ffdc64aff3641ba0cfaf00348e740b5802b0b8fb Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 18 Jun 2024 15:36:30 +0800 Subject: [PATCH 289/402] [VL] Set s.g.s.c.b.v.coalesceBatchesBeforeShuffle=true by default (#6056) --- .../FlushableHashAggregateRule.scala | 106 ++- .../v1-bhj-ras/spark32/1.txt | 100 +-- .../v1-bhj-ras/spark32/10.txt | 187 ++--- .../v1-bhj-ras/spark32/11.txt | 176 +++-- .../v1-bhj-ras/spark32/12.txt | 134 ++-- .../v1-bhj-ras/spark32/13.txt | 177 +++-- .../v1-bhj-ras/spark32/14.txt | 103 +-- .../v1-bhj-ras/spark32/15.txt | 146 ++-- .../v1-bhj-ras/spark32/16.txt | 193 ++--- .../v1-bhj-ras/spark32/17.txt | 125 +-- .../v1-bhj-ras/spark32/18.txt | 304 ++++---- .../v1-bhj-ras/spark32/19.txt | 99 +-- .../v1-bhj-ras/spark32/20.txt | 403 +++++----- .../v1-bhj-ras/spark32/21.txt | 245 +++--- .../v1-bhj-ras/spark32/22.txt | 122 +-- .../v1-bhj-ras/spark32/3.txt | 153 ++-- .../v1-bhj-ras/spark32/4.txt | 140 ++-- .../v1-bhj-ras/spark32/5.txt | 276 +++---- .../v1-bhj-ras/spark32/6.txt | 65 +- .../v1-bhj-ras/spark32/7.txt | 260 ++++--- .../v1-bhj-ras/spark32/8.txt | 348 ++++----- .../v1-bhj-ras/spark32/9.txt | 270 +++---- .../v1-bhj-ras/spark33/1.txt | 100 +-- .../v1-bhj-ras/spark33/10.txt | 187 ++--- .../v1-bhj-ras/spark33/11.txt | 341 +++++---- .../v1-bhj-ras/spark33/12.txt | 134 ++-- .../v1-bhj-ras/spark33/13.txt | 177 +++-- .../v1-bhj-ras/spark33/14.txt | 103 +-- .../v1-bhj-ras/spark33/15.txt | 239 +++--- .../v1-bhj-ras/spark33/16.txt | 193 ++--- .../v1-bhj-ras/spark33/17.txt | 125 +-- .../v1-bhj-ras/spark33/18.txt | 304 ++++---- .../v1-bhj-ras/spark33/19.txt | 99 +-- .../v1-bhj-ras/spark33/20.txt | 395 +++++----- .../v1-bhj-ras/spark33/21.txt | 241 +++--- .../v1-bhj-ras/spark33/22.txt | 197 ++--- .../v1-bhj-ras/spark33/3.txt | 153 ++-- .../v1-bhj-ras/spark33/4.txt | 140 ++-- .../v1-bhj-ras/spark33/5.txt | 276 +++---- .../v1-bhj-ras/spark33/6.txt | 65 +- .../v1-bhj-ras/spark33/7.txt | 260 ++++--- .../v1-bhj-ras/spark33/8.txt | 348 ++++----- .../v1-bhj-ras/spark33/9.txt | 270 +++---- .../v1-bhj-ras/spark34/1.txt | 100 +-- .../v1-bhj-ras/spark34/10.txt | 187 ++--- .../v1-bhj-ras/spark34/11.txt | 341 +++++---- .../v1-bhj-ras/spark34/12.txt | 134 ++-- .../v1-bhj-ras/spark34/13.txt | 177 +++-- .../v1-bhj-ras/spark34/14.txt | 103 +-- .../v1-bhj-ras/spark34/15.txt | 239 +++--- .../v1-bhj-ras/spark34/16.txt | 193 ++--- .../v1-bhj-ras/spark34/17.txt | 125 +-- .../v1-bhj-ras/spark34/18.txt | 304 ++++---- .../v1-bhj-ras/spark34/19.txt | 99 +-- .../v1-bhj-ras/spark34/20.txt | 312 ++++---- .../v1-bhj-ras/spark34/21.txt | 241 +++--- .../v1-bhj-ras/spark34/22.txt | 197 ++--- .../v1-bhj-ras/spark34/3.txt | 153 ++-- .../v1-bhj-ras/spark34/4.txt | 140 ++-- .../v1-bhj-ras/spark34/5.txt | 276 +++---- .../v1-bhj-ras/spark34/6.txt | 65 +- .../v1-bhj-ras/spark34/7.txt | 260 ++++--- .../v1-bhj-ras/spark34/8.txt | 348 ++++----- .../v1-bhj-ras/spark34/9.txt | 270 +++---- .../tpch-approved-plan/v1-bhj/spark32/1.txt | 100 +-- .../tpch-approved-plan/v1-bhj/spark32/10.txt | 187 ++--- .../tpch-approved-plan/v1-bhj/spark32/11.txt | 176 +++-- .../tpch-approved-plan/v1-bhj/spark32/12.txt | 134 ++-- .../tpch-approved-plan/v1-bhj/spark32/13.txt | 177 +++-- .../tpch-approved-plan/v1-bhj/spark32/14.txt | 103 +-- .../tpch-approved-plan/v1-bhj/spark32/15.txt | 146 ++-- .../tpch-approved-plan/v1-bhj/spark32/16.txt | 193 ++--- .../tpch-approved-plan/v1-bhj/spark32/17.txt | 125 +-- .../tpch-approved-plan/v1-bhj/spark32/18.txt | 304 ++++---- .../tpch-approved-plan/v1-bhj/spark32/19.txt | 99 +-- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 403 +++++----- .../tpch-approved-plan/v1-bhj/spark32/21.txt | 245 +++--- .../tpch-approved-plan/v1-bhj/spark32/22.txt | 122 +-- .../tpch-approved-plan/v1-bhj/spark32/3.txt | 153 ++-- .../tpch-approved-plan/v1-bhj/spark32/4.txt | 140 ++-- .../tpch-approved-plan/v1-bhj/spark32/5.txt | 276 +++---- .../tpch-approved-plan/v1-bhj/spark32/6.txt | 65 +- .../tpch-approved-plan/v1-bhj/spark32/7.txt | 260 ++++--- .../tpch-approved-plan/v1-bhj/spark32/8.txt | 348 ++++----- .../tpch-approved-plan/v1-bhj/spark32/9.txt | 270 +++---- .../tpch-approved-plan/v1-bhj/spark33/1.txt | 100 +-- .../tpch-approved-plan/v1-bhj/spark33/10.txt | 187 ++--- .../tpch-approved-plan/v1-bhj/spark33/11.txt | 341 +++++---- .../tpch-approved-plan/v1-bhj/spark33/12.txt | 134 ++-- .../tpch-approved-plan/v1-bhj/spark33/13.txt | 177 +++-- .../tpch-approved-plan/v1-bhj/spark33/14.txt | 103 +-- .../tpch-approved-plan/v1-bhj/spark33/15.txt | 239 +++--- .../tpch-approved-plan/v1-bhj/spark33/16.txt | 193 ++--- .../tpch-approved-plan/v1-bhj/spark33/17.txt | 125 +-- .../tpch-approved-plan/v1-bhj/spark33/18.txt | 304 ++++---- .../tpch-approved-plan/v1-bhj/spark33/19.txt | 99 +-- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 395 +++++----- .../tpch-approved-plan/v1-bhj/spark33/21.txt | 241 +++--- .../tpch-approved-plan/v1-bhj/spark33/22.txt | 230 +++--- .../tpch-approved-plan/v1-bhj/spark33/3.txt | 153 ++-- .../tpch-approved-plan/v1-bhj/spark33/4.txt | 140 ++-- .../tpch-approved-plan/v1-bhj/spark33/5.txt | 276 +++---- .../tpch-approved-plan/v1-bhj/spark33/6.txt | 65 +- .../tpch-approved-plan/v1-bhj/spark33/7.txt | 260 ++++--- .../tpch-approved-plan/v1-bhj/spark33/8.txt | 348 ++++----- .../tpch-approved-plan/v1-bhj/spark33/9.txt | 270 +++---- .../tpch-approved-plan/v1-bhj/spark34/1.txt | 100 +-- .../tpch-approved-plan/v1-bhj/spark34/10.txt | 187 ++--- .../tpch-approved-plan/v1-bhj/spark34/11.txt | 341 +++++---- .../tpch-approved-plan/v1-bhj/spark34/12.txt | 134 ++-- .../tpch-approved-plan/v1-bhj/spark34/13.txt | 177 +++-- .../tpch-approved-plan/v1-bhj/spark34/14.txt | 103 +-- .../tpch-approved-plan/v1-bhj/spark34/15.txt | 239 +++--- .../tpch-approved-plan/v1-bhj/spark34/16.txt | 193 ++--- .../tpch-approved-plan/v1-bhj/spark34/17.txt | 125 +-- .../tpch-approved-plan/v1-bhj/spark34/18.txt | 304 ++++---- .../tpch-approved-plan/v1-bhj/spark34/19.txt | 99 +-- .../tpch-approved-plan/v1-bhj/spark34/20.txt | 312 ++++---- .../tpch-approved-plan/v1-bhj/spark34/21.txt | 241 +++--- .../tpch-approved-plan/v1-bhj/spark34/22.txt | 230 +++--- .../tpch-approved-plan/v1-bhj/spark34/3.txt | 153 ++-- .../tpch-approved-plan/v1-bhj/spark34/4.txt | 140 ++-- .../tpch-approved-plan/v1-bhj/spark34/5.txt | 276 +++---- .../tpch-approved-plan/v1-bhj/spark34/6.txt | 65 +- .../tpch-approved-plan/v1-bhj/spark34/7.txt | 260 ++++--- .../tpch-approved-plan/v1-bhj/spark34/8.txt | 348 ++++----- .../tpch-approved-plan/v1-bhj/spark34/9.txt | 270 +++---- .../tpch-approved-plan/v1-ras/spark32/1.txt | 100 +-- .../tpch-approved-plan/v1-ras/spark32/10.txt | 345 +++++---- .../tpch-approved-plan/v1-ras/spark32/11.txt | 284 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 204 ++--- .../tpch-approved-plan/v1-ras/spark32/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark32/15.txt | 173 +++-- .../tpch-approved-plan/v1-ras/spark32/16.txt | 251 +++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 382 +++++----- .../tpch-approved-plan/v1-ras/spark32/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark32/20.txt | 502 ++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 476 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 178 +++-- .../tpch-approved-plan/v1-ras/spark32/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 194 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 538 +++++++------ .../tpch-approved-plan/v1-ras/spark32/6.txt | 65 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 511 +++++++------ .../tpch-approved-plan/v1-ras/spark32/8.txt | 714 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 534 +++++++------ .../tpch-approved-plan/v1-ras/spark33/1.txt | 100 +-- .../tpch-approved-plan/v1-ras/spark33/10.txt | 345 +++++---- .../tpch-approved-plan/v1-ras/spark33/11.txt | 478 ++++++------ .../tpch-approved-plan/v1-ras/spark33/12.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 204 ++--- .../tpch-approved-plan/v1-ras/spark33/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark33/15.txt | 266 ++++--- .../tpch-approved-plan/v1-ras/spark33/16.txt | 251 +++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 382 +++++----- .../tpch-approved-plan/v1-ras/spark33/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark33/20.txt | 494 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 472 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 253 ++++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 194 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 538 +++++++------ .../tpch-approved-plan/v1-ras/spark33/6.txt | 65 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 511 +++++++------ .../tpch-approved-plan/v1-ras/spark33/8.txt | 714 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 534 +++++++------ .../tpch-approved-plan/v1-ras/spark34/1.txt | 100 +-- .../tpch-approved-plan/v1-ras/spark34/10.txt | 345 +++++---- .../tpch-approved-plan/v1-ras/spark34/11.txt | 478 ++++++------ .../tpch-approved-plan/v1-ras/spark34/12.txt | 190 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 204 ++--- .../tpch-approved-plan/v1-ras/spark34/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark34/15.txt | 266 ++++--- .../tpch-approved-plan/v1-ras/spark34/16.txt | 251 +++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 221 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 382 +++++----- .../tpch-approved-plan/v1-ras/spark34/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark34/20.txt | 494 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 472 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 253 ++++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 194 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 538 +++++++------ .../tpch-approved-plan/v1-ras/spark34/6.txt | 65 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 511 +++++++------ .../tpch-approved-plan/v1-ras/spark34/8.txt | 714 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 534 +++++++------ .../tpch-approved-plan/v1/spark32/1.txt | 100 +-- .../tpch-approved-plan/v1/spark32/10.txt | 345 +++++---- .../tpch-approved-plan/v1/spark32/11.txt | 284 +++---- .../tpch-approved-plan/v1/spark32/12.txt | 190 ++--- .../tpch-approved-plan/v1/spark32/13.txt | 204 ++--- .../tpch-approved-plan/v1/spark32/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark32/15.txt | 173 +++-- .../tpch-approved-plan/v1/spark32/16.txt | 251 +++--- .../tpch-approved-plan/v1/spark32/17.txt | 221 +++--- .../tpch-approved-plan/v1/spark32/18.txt | 382 +++++----- .../tpch-approved-plan/v1/spark32/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark32/20.txt | 502 ++++++------ .../tpch-approved-plan/v1/spark32/21.txt | 476 ++++++------ .../tpch-approved-plan/v1/spark32/22.txt | 178 +++-- .../tpch-approved-plan/v1/spark32/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark32/4.txt | 194 ++--- .../tpch-approved-plan/v1/spark32/5.txt | 538 +++++++------ .../tpch-approved-plan/v1/spark32/6.txt | 65 +- .../tpch-approved-plan/v1/spark32/7.txt | 511 +++++++------ .../tpch-approved-plan/v1/spark32/8.txt | 714 ++++++++++-------- .../tpch-approved-plan/v1/spark32/9.txt | 534 +++++++------ .../tpch-approved-plan/v1/spark33/1.txt | 100 +-- .../tpch-approved-plan/v1/spark33/10.txt | 345 +++++---- .../tpch-approved-plan/v1/spark33/11.txt | 478 ++++++------ .../tpch-approved-plan/v1/spark33/12.txt | 190 ++--- .../tpch-approved-plan/v1/spark33/13.txt | 204 ++--- .../tpch-approved-plan/v1/spark33/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark33/15.txt | 266 ++++--- .../tpch-approved-plan/v1/spark33/16.txt | 251 +++--- .../tpch-approved-plan/v1/spark33/17.txt | 221 +++--- .../tpch-approved-plan/v1/spark33/18.txt | 382 +++++----- .../tpch-approved-plan/v1/spark33/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark33/20.txt | 494 ++++++------ .../tpch-approved-plan/v1/spark33/21.txt | 472 ++++++------ .../tpch-approved-plan/v1/spark33/22.txt | 286 +++---- .../tpch-approved-plan/v1/spark33/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark33/4.txt | 194 ++--- .../tpch-approved-plan/v1/spark33/5.txt | 538 +++++++------ .../tpch-approved-plan/v1/spark33/6.txt | 65 +- .../tpch-approved-plan/v1/spark33/7.txt | 511 +++++++------ .../tpch-approved-plan/v1/spark33/8.txt | 714 ++++++++++-------- .../tpch-approved-plan/v1/spark33/9.txt | 534 +++++++------ .../tpch-approved-plan/v1/spark34/1.txt | 100 +-- .../tpch-approved-plan/v1/spark34/10.txt | 345 +++++---- .../tpch-approved-plan/v1/spark34/11.txt | 478 ++++++------ .../tpch-approved-plan/v1/spark34/12.txt | 190 ++--- .../tpch-approved-plan/v1/spark34/13.txt | 204 ++--- .../tpch-approved-plan/v1/spark34/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark34/15.txt | 266 ++++--- .../tpch-approved-plan/v1/spark34/16.txt | 251 +++--- .../tpch-approved-plan/v1/spark34/17.txt | 221 +++--- .../tpch-approved-plan/v1/spark34/18.txt | 382 +++++----- .../tpch-approved-plan/v1/spark34/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark34/20.txt | 494 ++++++------ .../tpch-approved-plan/v1/spark34/21.txt | 472 ++++++------ .../tpch-approved-plan/v1/spark34/22.txt | 286 +++---- .../tpch-approved-plan/v1/spark34/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark34/4.txt | 194 ++--- .../tpch-approved-plan/v1/spark34/5.txt | 538 +++++++------ .../tpch-approved-plan/v1/spark34/6.txt | 65 +- .../tpch-approved-plan/v1/spark34/7.txt | 511 +++++++------ .../tpch-approved-plan/v1/spark34/8.txt | 714 ++++++++++-------- .../tpch-approved-plan/v1/spark34/9.txt | 534 +++++++------ .../spark/sql/GlutenImplicitsTest.scala | 8 +- .../GlutenQueryExecutionErrorsSuite.scala | 9 + .../org/apache/gluten/GlutenConfig.scala | 11 +- 256 files changed, 35079 insertions(+), 29973 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index f850b6f457ea..777bf553856e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.execution.{FlushableHashAggregateExecTransformer, HashAggregateExecTransformer, ProjectExecTransformer, RegularHashAggregateExecTransformer} +import org.apache.gluten.execution._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.aggregate.{Partial, PartialMerge} @@ -30,74 +30,55 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike * optimizations such as flushing and abandoning. */ case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkPlan] { + import FlushableHashAggregateRule._ override def apply(plan: SparkPlan): SparkPlan = plan.transformUp { - case shuffle: ShuffleExchangeLike => + case s: ShuffleExchangeLike => // If an exchange follows a hash aggregate in which all functions are in partial mode, // then it's safe to convert the hash aggregate to flushable hash aggregate. - shuffle.child match { - case HashAggPropagatedToShuffle(proj, agg) => - shuffle.withNewChildren( - Seq(proj.withNewChildren(Seq(FlushableHashAggregateExecTransformer( - agg.requiredChildDistributionExpressions, - agg.groupingExpressions, - agg.aggregateExpressions, - agg.aggregateAttributes, - agg.initialInputBufferOffset, - agg.resultExpressions, - agg.child - ))))) - case HashAggWithShuffle(agg) => - shuffle.withNewChildren( - Seq(FlushableHashAggregateExecTransformer( - agg.requiredChildDistributionExpressions, - agg.groupingExpressions, - agg.aggregateExpressions, - agg.aggregateAttributes, - agg.initialInputBufferOffset, - agg.resultExpressions, - agg.child - ))) - case _ => - shuffle - } + val out = s.withNewChildren( + List( + replaceEligibleAggregates(s.child) { + agg => + FlushableHashAggregateExecTransformer( + agg.requiredChildDistributionExpressions, + agg.groupingExpressions, + agg.aggregateExpressions, + agg.aggregateAttributes, + agg.initialInputBufferOffset, + agg.resultExpressions, + agg.child + ) + } + ) + ) + out } -} -object HashAggPropagatedToShuffle { - def unapply( - plan: SparkPlan): Option[(ProjectExecTransformer, RegularHashAggregateExecTransformer)] = { - if (!plan.isInstanceOf[ProjectExecTransformer]) { - return None - } - val proj = plan.asInstanceOf[ProjectExecTransformer] - val child = proj.child - if (!child.isInstanceOf[RegularHashAggregateExecTransformer]) { - return None - } - val agg = child.asInstanceOf[RegularHashAggregateExecTransformer] - if (!agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge)) { - return None + private def replaceEligibleAggregates(plan: SparkPlan)( + func: RegularHashAggregateExecTransformer => SparkPlan): SparkPlan = { + def transformDown: SparkPlan => SparkPlan = { + case agg: RegularHashAggregateExecTransformer + if !agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge) => + // Not a intermediate agg. Skip. + agg + case agg: RegularHashAggregateExecTransformer + if isAggInputAlreadyDistributedWithAggKeys(agg) => + // Data already grouped by aggregate keys, Skip. + agg + case agg: RegularHashAggregateExecTransformer => + func(agg) + case p if !canPropagate(p) => p + case other => other.withNewChildren(other.children.map(transformDown)) } - if (FlushableHashAggregateRule.isAggInputAlreadyDistributedWithAggKeys(agg)) { - return None - } - Some((proj, agg)) + + val out = transformDown(plan) + out } -} -object HashAggWithShuffle { - def unapply(plan: SparkPlan): Option[RegularHashAggregateExecTransformer] = { - if (!plan.isInstanceOf[RegularHashAggregateExecTransformer]) { - return None - } - val agg = plan.asInstanceOf[RegularHashAggregateExecTransformer] - if (!agg.aggregateExpressions.forall(p => p.mode == Partial || p.mode == PartialMerge)) { - return None - } - if (FlushableHashAggregateRule.isAggInputAlreadyDistributedWithAggKeys(agg)) { - return None - } - Some(agg) + private def canPropagate(plan: SparkPlan): Boolean = plan match { + case _: ProjectExecTransformer => true + case _: VeloxAppendBatchesExec => true + case _ => false } } @@ -112,7 +93,8 @@ object FlushableHashAggregateRule { * only on a single partition among the whole cluster. Spark's planner may use this information to * perform optimizations like doing "partial_count(a, b, c)" directly on the output data. */ - def isAggInputAlreadyDistributedWithAggKeys(agg: HashAggregateExecTransformer): Boolean = { + private def isAggInputAlreadyDistributedWithAggKeys( + agg: HashAggregateExecTransformer): Boolean = { if (agg.groupingExpressions.isEmpty) { // Empty grouping set () should not be satisfied by any partitioning patterns. // E.g., diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 9316f6f8ebb5..53edb933c1fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index d67e373daf9b..29f28ac26a6b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index f65747c37e65..b8f466eb9456 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,55 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -158,153 +160,161 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index f1b7bbfe6acb..1e60a93910a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,41 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -106,121 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(23) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 12147d4b3197..c525944e1262 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -109,174 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index 87dc45754b66..fbfcf07030eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index 0bfcd38f02d2..b5073901c620 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- Project (43) + +- BroadcastHashJoin Inner BuildLeft (42) + :- BroadcastExchange (34) + : +- Filter (33) + : +- Scan parquet (32) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -99,138 +101,146 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(32) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(34) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(43) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(44) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index ccc5488215a6..f3cef927551c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,53 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -114,195 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt index 9d81e1d5053f..7c5359849d4e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index e582fd47cf55..a0a6e26a6ac3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 72a9f74f86ce..d2ab9979f8a3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index e09911ea6ad8..6ebe36be3494 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,97 +1,100 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (110) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (73) + +- ^ SortExecTransformer (71) + +- ^ InputIteratorTransformer (70) + +- ShuffleQueryStage (68) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ NoopFilter (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ NoopFilter (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ NoopFilter (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (109) + +- Exchange (108) + +- Project (107) + +- BroadcastHashJoin Inner BuildRight (106) + :- Project (101) + : +- ShuffledHashJoin LeftSemi BuildRight (100) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Exchange (99) + : +- Project (98) + : +- BroadcastHashJoin Inner BuildLeft (97) + : :- BroadcastExchange (84) + : : +- BroadcastHashJoin LeftSemi BuildRight (83) + : : :- Filter (78) + : : : +- Scan parquet (77) + : : +- BroadcastExchange (82) + : : +- Project (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- Filter (96) + : +- HashAggregate (95) + : +- Exchange (94) + : +- HashAggregate (93) + : +- BroadcastHashJoin LeftSemi BuildRight (92) + : :- Project (87) + : : +- Filter (86) + : : +- Scan parquet (85) + : +- BroadcastExchange (91) + : +- Project (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (105) + +- Project (104) + +- Filter (103) + +- Scan parquet (102) (1) Scan parquet @@ -113,448 +116,460 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(12) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) NoopFilter +(14) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) NoopFilter +(28) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) NoopFilter +(56) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(55) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(66) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(69) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(71) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(73) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(74) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(75) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(76) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(77) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(78) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(79) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(80) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(81) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(82) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(84) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(86) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(87) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(88) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(89) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(90) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(91) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(93) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(94) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(95) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(96) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(98) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(99) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(101) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(102) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(103) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(104) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(105) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(107) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(108) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(109) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index 10e08cfd60b5..eb3f64d3efcc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -301,194 +302,198 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index a8c66bef1716..4c149d9bb43e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,37 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -93,112 +95,120 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(21) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index b9b7951d7652..df2a6de8d184 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 97d18724dc9d..34cb6467cbc2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -112,125 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index a9b993d0a078..2c9ba4c938d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,91 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -312,221 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(60) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index 7aae3dccfd9b..b6f876d48e5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index b0d68672ca3a..8cf20f7694c9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,85 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -282,213 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(55) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 7e20b94edaa7..045e283bd036 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,116 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -413,273 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(79) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index 8a30fbdfb152..abdd1d7b9369 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,89 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -306,217 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(59) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index 63b30bb5d26b..afea15af53d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 18c2ef9bc473..7a509e435d07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 6afbee587ed2..1c4e102aa0b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,55 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -158,379 +160,392 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ NoopFilter (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) NoopFilter +(62) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(61) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index c65916d66898..12d0f6f0ff95 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,41 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -106,121 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(23) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index 9ff054f9314e..1dfb80d78050 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -109,174 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index cc14a0347f1d..6f71859cba6d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 2d463a561ab6..40b342510de2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -98,284 +100,297 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(42) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) NoopFilter +(45) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(44) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index c4d21c42e70b..d579c1a704e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,53 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -114,195 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt index 0b41400d5864..1e2ed970aef4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 8f03bd54cd2d..2e7ce455aebd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 1fcd294716ef..992c5328adba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index ee30d646767e..27e59afbb7fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,96 +1,99 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (107) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (70) + +- AQEShuffleRead (69) + +- ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50), Statistics(X) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24), Statistics(X) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ NoopFilter (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18), Statistics(X) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ NoopFilter (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40), Statistics(X) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60), Statistics(X) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ NoopFilter (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (106) + +- Exchange (105) + +- Project (104) + +- BroadcastHashJoin Inner BuildRight (103) + :- Project (98) + : +- ShuffledHashJoin LeftSemi BuildRight (97) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (96) + : +- Project (95) + : +- BroadcastHashJoin Inner BuildLeft (94) + : :- BroadcastExchange (81) + : : +- BroadcastHashJoin LeftSemi BuildRight (80) + : : :- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (79) + : : +- Project (78) + : : +- Filter (77) + : : +- Scan parquet (76) + : +- Filter (93) + : +- HashAggregate (92) + : +- Exchange (91) + : +- HashAggregate (90) + : +- BroadcastHashJoin LeftSemi BuildRight (89) + : :- Project (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- BroadcastExchange (88) + : +- Project (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (102) + +- Project (101) + +- Filter (100) + +- Scan parquet (99) (1) Scan parquet @@ -112,438 +115,450 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(12) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) NoopFilter +(14) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) NoopFilter +(28) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) NoopFilter +(56) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(55) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(66) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(69) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(71) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(72) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(73) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(74) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(75) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(84) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(90) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(91) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(92) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(93) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(95) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(96) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(98) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(99) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(100) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(101) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(102) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(104) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(105) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(106) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(107) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index c52c51139d2a..7627c7f4f147 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -300,190 +301,194 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index d404e2149a89..9a0475d25d78 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,37 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -93,226 +95,239 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(21) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ NoopFilter (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(39) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) NoopFilter +(42) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(41) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index dbe67ba34d04..60843c0991ad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 626f608cecd0..b142ee1d8f33 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -112,125 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index 7a2ce7cf7a4a..c8a62c3aca1f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,91 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -312,221 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(60) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index e03830992c2e..fa9c936a0ca1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 4f9b5f20956d..f098b9ba837f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,85 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -282,213 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(55) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 677a42eb711a..becfc12a1b86 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,116 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -413,273 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(79) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 9f2111b015ca..a486fc65677b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,89 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -306,217 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(59) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index c254ec8c82ca..545f2e7e086d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index ffc37f5b7908..9325f007789b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -212,158 +213,162 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index 6410bb3fed5f..c593374b9a00 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,55 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -160,385 +162,398 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ NoopFilter (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) NoopFilter +(62) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(61) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index 4e85d8194098..93f3a4cf9ab7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,41 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -107,122 +109,130 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(23) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index e83ab1aca83c..e44b10042401 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -110,175 +113,187 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index 58d532222af5..8f5ae0f52e63 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -97,96 +98,100 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 269921602469..d616981e57d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -98,286 +100,299 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(42) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) NoopFilter +(45) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(44) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index 0a8771619020..3b4da47fc25f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,53 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -115,197 +118,209 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt index 62612096db42..77733ad6f8a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -58,143 +59,147 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index f0420b98194d..0f5e82ec54c1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -138,341 +140,349 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index c39011756bca..3bafdb994153 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -96,92 +97,96 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index 6bad140301d8..945cc79de1ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,90 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ NoopFilter (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ NoopFilter (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (62) + +- AQEShuffleRead (61) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (55) + :- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (45) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (44) + : +- BroadcastQueryStage (42), Statistics(X) + : +- ColumnarBroadcastExchange (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (38) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ NoopFilter (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ NoopFilter (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (37) + : +- ^ ProjectExecTransformer (36) + : +- ^ RegularHashAggregateExecTransformer (35) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- VeloxAppendBatches (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (54) + +- BroadcastQueryStage (52), Statistics(X) + +- ColumnarBroadcastExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ NoopFilter (48) + +- ^ Scan parquet (47) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (97) + +- Exchange (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (87) + : +- Project (86) + : +- BroadcastHashJoin Inner BuildLeft (85) + : :- BroadcastExchange (72) + : : +- BroadcastHashJoin LeftSemi BuildRight (71) + : : :- Filter (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (70) + : : +- Project (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Filter (84) + : +- HashAggregate (83) + : +- Exchange (82) + : +- HashAggregate (81) + : +- BroadcastHashJoin LeftSemi BuildRight (80) + : :- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -215,309 +217,317 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(31) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(36) FilterExecTransformer +(37) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(37) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(38) ProjectExecTransformer +(39) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(40) ColumnarBroadcastExchange +(41) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(41) BroadcastQueryStage +(42) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(42) InputAdapter +(43) InputAdapter Input [1]: [ps_suppkey#X] -(43) InputIteratorTransformer +(44) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(47) NoopFilter +(48) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(50) ColumnarBroadcastExchange +(51) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastQueryStage +(52) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(52) InputAdapter +(53) InputAdapter Input [1]: [n_nationkey#X] -(53) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [n_nationkey#X] -(54) BroadcastHashJoinExecTransformer +(55) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(58) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(59) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(61) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(63) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(65) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(66) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(72) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(73) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(75) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(81) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(82) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(83) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(84) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(86) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(87) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(89) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(95) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(96) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(97) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 14b23aa966e5..a7d9f42063a0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -305,195 +306,199 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 4517787bcaef..0d9139bcd990 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,37 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -94,227 +96,240 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(21) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ NoopFilter (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(39) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) NoopFilter +(42) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(41) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index 361ea070d42a..988bdc6c26ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -161,133 +162,137 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 4bdcd640058c..0b73608fbe4b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -113,126 +115,134 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index d1f0cb5eba56..8891188e8a08 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,91 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -317,226 +319,234 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(60) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 5987a808f5fd..64624c791f72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 29c2524a7615..ae1ab637f805 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,85 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -287,218 +289,226 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(55) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index 715bc6f4e5f8..4bbf7967744c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,116 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -420,280 +422,288 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(79) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 14c32b04eb48..301c001eb793 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,89 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -311,222 +313,230 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(59) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt index 656f98574483..63b7d317f3cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index 3afadbc4c678..db7c1cb79667 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt index 03199cd9feb1..33c7971d3749 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt @@ -1,55 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -158,153 +160,161 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt index e78891a522ff..f1f3a9234354 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt @@ -1,41 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -106,121 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(23) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt index 9aa658b2d78d..13ef25b681c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -109,174 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt index fcd550bdb92c..c86f2215cda5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt index 54dad76174f9..8edb179e592f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- Project (43) + +- BroadcastHashJoin Inner BuildLeft (42) + :- BroadcastExchange (34) + : +- Filter (33) + : +- Scan parquet (32) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -99,138 +101,146 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(32) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(34) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(43) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(44) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt index 5197f57218a4..4d4d52c7d2e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt @@ -1,53 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -114,195 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt index 9d81e1d5053f..7c5359849d4e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index 341589565740..a4aaf08ff8bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt index a46daacbdb75..87acf3c4b28e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index 6fcedbbda996..7b494469aacc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -1,97 +1,100 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (110) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (73) + +- ^ SortExecTransformer (71) + +- ^ InputIteratorTransformer (70) + +- ShuffleQueryStage (68) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ FilterExecTransformer (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ FilterExecTransformer (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ FilterExecTransformer (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (109) + +- Exchange (108) + +- Project (107) + +- BroadcastHashJoin Inner BuildRight (106) + :- Project (101) + : +- ShuffledHashJoin LeftSemi BuildRight (100) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Exchange (99) + : +- Project (98) + : +- BroadcastHashJoin Inner BuildLeft (97) + : :- BroadcastExchange (84) + : : +- BroadcastHashJoin LeftSemi BuildRight (83) + : : :- Filter (78) + : : : +- Scan parquet (77) + : : +- BroadcastExchange (82) + : : +- Project (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- Filter (96) + : +- HashAggregate (95) + : +- Exchange (94) + : +- HashAggregate (93) + : +- BroadcastHashJoin LeftSemi BuildRight (92) + : :- Project (87) + : : +- Filter (86) + : : +- Scan parquet (85) + : +- BroadcastExchange (91) + : +- Project (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (105) + +- Project (104) + +- Filter (103) + +- Scan parquet (102) (1) Scan parquet @@ -113,448 +116,460 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer +(12) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer +(14) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer +(56) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(55) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(66) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(69) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(71) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(73) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(74) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(75) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(76) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(77) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(78) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(79) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(80) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(81) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(82) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(84) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(85) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(86) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(87) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(88) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(89) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(90) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(91) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(93) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(94) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(95) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(96) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(98) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(99) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(101) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(102) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(103) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(104) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(105) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(106) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(107) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(108) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(109) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index 4a970927299d..ee28d7b592e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -301,194 +302,198 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt index d0252d7ac997..d578b43f3d6a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt @@ -1,37 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -93,112 +95,120 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(21) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index ba63341e7e03..2ffd75b92964 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt index 54c5c1b24d8b..36c6de5374ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -112,125 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt index 2f037bd9c1d1..49ee27485b4a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt @@ -1,91 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -312,221 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(60) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt index b39d8c1b2aec..786a89fe715a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt index 4b2aa1e644a2..2ba42f806f3e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt @@ -1,85 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -282,213 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(55) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt index fe9d78579598..64a40563c238 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt @@ -1,116 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -413,273 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(79) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt index cb62cd97d3e0..b5a21a0261d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt @@ -1,89 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -306,217 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(59) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt index 89de3133895b..22dd5100c4fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index 898458b34cb7..8681ecf5f93f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt index 6677083c130d..6dfc1d00f4cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt @@ -1,55 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -158,379 +160,392 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ FilterExecTransformer (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(61) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt index e83423c0b2f2..1e1ad6d497a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt @@ -1,41 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -106,121 +108,129 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(23) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt index 1fa2d94b096b..ff6532aa6579 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -109,174 +112,186 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt index 5bc1aef67790..d144dd39ca1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt index 5cd45806c9fb..c4f825f5ca53 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -98,284 +100,297 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(42) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer +(45) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(44) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt index 56d39742c76e..741b837976a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt @@ -1,53 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -114,195 +117,207 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt index 0b41400d5864..1e2ed970aef4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index c37f0b49567c..05884fea7885 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt index 6822887d3aed..f04466031352 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index 32f28a0311c4..a0edead7013d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -1,96 +1,99 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (107) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (70) + +- AQEShuffleRead (69) + +- ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- VeloxAppendBatches (66) + +- ^ ProjectExecTransformer (64) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (63) + :- ^ ProjectExecTransformer (54) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (53) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (52) + : +- BroadcastQueryStage (50), Statistics(X) + : +- ColumnarBroadcastExchange (49) + : +- ^ ProjectExecTransformer (47) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (46) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24), Statistics(X) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ FilterExecTransformer (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18), Statistics(X) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ FilterExecTransformer (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (45) + : +- ^ ProjectExecTransformer (44) + : +- ^ RegularHashAggregateExecTransformer (43) + : +- ^ InputIteratorTransformer (42) + : +- ShuffleQueryStage (40), Statistics(X) + : +- ColumnarExchange (39) + : +- VeloxAppendBatches (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (62) + +- BroadcastQueryStage (60), Statistics(X) + +- ColumnarBroadcastExchange (59) + +- ^ ProjectExecTransformer (57) + +- ^ FilterExecTransformer (56) + +- ^ Scan parquet (55) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (106) + +- Exchange (105) + +- Project (104) + +- BroadcastHashJoin Inner BuildRight (103) + :- Project (98) + : +- ShuffledHashJoin LeftSemi BuildRight (97) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (96) + : +- Project (95) + : +- BroadcastHashJoin Inner BuildLeft (94) + : :- BroadcastExchange (81) + : : +- BroadcastHashJoin LeftSemi BuildRight (80) + : : :- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (79) + : : +- Project (78) + : : +- Filter (77) + : : +- Scan parquet (76) + : +- Filter (93) + : +- HashAggregate (92) + : +- Exchange (91) + : +- HashAggregate (90) + : +- BroadcastHashJoin LeftSemi BuildRight (89) + : :- Project (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- BroadcastExchange (88) + : +- Project (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (102) + +- Project (101) + +- Filter (100) + +- Scan parquet (99) (1) Scan parquet @@ -112,438 +115,450 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer +(12) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer +(14) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(39) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(41) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(43) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(46) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(49) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(50) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(51) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(52) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(53) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(54) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(55) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer +(56) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(55) ProjectExecTransformer +(57) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(59) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(60) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(63) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(66) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(67) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(69) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(71) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(72) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(73) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(74) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(75) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(82) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(83) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(84) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(88) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(90) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(91) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(92) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(93) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(95) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(96) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(98) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(99) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(100) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(101) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(102) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(104) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(105) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(106) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(107) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt index 252a06b11fa7..3ffdec004af2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -300,190 +301,194 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt index 5a6f0dc8fe05..f30752effaa4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt @@ -1,37 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -93,246 +95,260 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(21) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(39) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(41) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 722def52f850..d9f87ca9b538 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt index 0ca16a6174b7..db9f7716a365 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -112,125 +114,133 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt index 24ba48c495f6..67e9d847aefa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt @@ -1,91 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -312,221 +314,229 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(60) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt index 68854bdea473..3432579a0de0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt index fd1e1e8fa37f..3db2d7669f14 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt @@ -1,85 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -282,213 +284,221 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(55) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt index fc7a79a5b2b3..f77b5fca35c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt @@ -1,116 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -413,273 +415,281 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(79) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt index 4f18dfa35261..1be487ed123a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt @@ -1,89 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -306,217 +308,225 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(59) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt index 090a9522f13a..1e53cd90e1b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index 808f58b58fa5..098b6610a2e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -212,158 +213,162 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt index e6f8232f12c4..725debe00d5e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt @@ -1,55 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ FilterExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (59) + +- Exchange (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- Project (47) + : +- BroadcastHashJoin Inner BuildRight (46) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (45) + : +- Filter (44) + : +- Scan parquet (43) + +- BroadcastExchange (51) + +- Project (50) + +- Filter (49) + +- Scan parquet (48) (1) Scan parquet @@ -160,385 +162,398 @@ Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(25) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(26) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(28) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(36) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(38) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(41) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(42) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(45) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(47) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(48) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(49) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(50) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(51) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(53) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(54) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(55) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(57) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(58) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(59) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +Subquery:1 Hosting operator id = 31 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (85) + +- ^ ProjectExecTransformer (83) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- VeloxAppendBatches (77) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (73) + :- ^ ProjectExecTransformer (68) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (67) + : :- ^ FilterExecTransformer (62) + : : +- ^ Scan parquet (61) + : +- ^ InputIteratorTransformer (66) + : +- BroadcastQueryStage (64), Statistics(X) + : +- ReusedExchange (63) + +- ^ InputIteratorTransformer (72) + +- BroadcastQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (101) + +- Exchange (100) + +- HashAggregate (99) + +- Project (98) + +- BroadcastHashJoin Inner BuildRight (97) + :- Project (92) + : +- BroadcastHashJoin Inner BuildRight (91) + : :- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- BroadcastExchange (96) + +- Project (95) + +- Filter (94) + +- Scan parquet (93) + + +(61) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(61) ReusedExchange [Reuses operator id: 6] +(63) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(64) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(65) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(66) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(69) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(70) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(71) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(72) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(73) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(74) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(76) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(77) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(78) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(80) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(83) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(86) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(87) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(88) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(89) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(90) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(92) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(93) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(94) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(95) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(96) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(98) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(99) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(100) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(101) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt index 261e4fc821ea..e2b9e5153009 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt @@ -1,41 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- BroadcastHashJoin Inner BuildLeft (37) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -107,122 +109,130 @@ Input [3]: [l_shipmode#X, sum#X, sum#X] Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(23) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(24) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(28) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(31) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(32) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(33) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(34) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(35) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(36) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(37) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(38) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(40) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(42) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt index 7ccf5dafe28e..f64de4dee4b2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- BroadcastHashJoin LeftOuter BuildRight (45) + :- Scan parquet (40) + +- BroadcastExchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -110,175 +113,187 @@ Input [2]: [c_custkey#X, count#X] Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, count#X] +Arguments: X + +(16) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(22) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(41) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(43) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(48) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt index 8655ec75dd6f..901c481fac69 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -97,96 +98,100 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt index 0853cd6948f9..a87fdff4537d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt @@ -1,42 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (21) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (42) + +- Exchange (41) + +- Project (40) + +- BroadcastHashJoin Inner BuildLeft (39) + :- BroadcastExchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -98,286 +100,299 @@ Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(27) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(31) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- VeloxAppendBatches (50) + +- ^ ProjectExecTransformer (48) + +- ^ FlushableHashAggregateExecTransformer (47) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(42) Scan parquet +(44) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer +(45) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(44) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(47) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(50) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt index 78c65e72e6c8..e2f073b8908f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt @@ -1,53 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (58) + +- Exchange (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- BroadcastHashJoin Inner BuildRight (49) + :- BroadcastHashJoin LeftAnti BuildRight (45) + : :- Filter (40) + : : +- Scan parquet (39) + : +- BroadcastExchange (44) + : +- Project (43) + : +- Filter (42) + : +- Scan parquet (41) + +- BroadcastExchange (48) + +- Filter (47) + +- Scan parquet (46) (1) Scan parquet @@ -115,197 +118,209 @@ Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(16) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(36) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(39) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(40) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(41) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(42) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(43) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(44) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(46) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(48) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(50) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(51) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(52) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(53) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(57) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(58) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt index 62612096db42..77733ad6f8a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ FlushableHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -58,143 +59,147 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 58751c9b885f..4f36a185da72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -138,341 +140,349 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt index e2f0c051b69a..7854a767b26b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -96,92 +97,96 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt index 3b7e5334a3ba..981017da501e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt @@ -1,90 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ FilterExecTransformer (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ FilterExecTransformer (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (62) + +- AQEShuffleRead (61) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (55) + :- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (45) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (44) + : +- BroadcastQueryStage (42), Statistics(X) + : +- ColumnarBroadcastExchange (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (38) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ FilterExecTransformer (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ FilterExecTransformer (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (37) + : +- ^ ProjectExecTransformer (36) + : +- ^ RegularHashAggregateExecTransformer (35) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- VeloxAppendBatches (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (54) + +- BroadcastQueryStage (52), Statistics(X) + +- ColumnarBroadcastExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FilterExecTransformer (48) + +- ^ Scan parquet (47) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (97) + +- Exchange (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (87) + : +- Project (86) + : +- BroadcastHashJoin Inner BuildLeft (85) + : :- BroadcastExchange (72) + : : +- BroadcastHashJoin LeftSemi BuildRight (71) + : : :- Filter (66) + : : : +- Scan parquet (65) + : : +- BroadcastExchange (70) + : : +- Project (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Filter (84) + : +- HashAggregate (83) + : +- Exchange (82) + : +- HashAggregate (81) + : +- BroadcastHashJoin LeftSemi BuildRight (80) + : :- Project (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -215,309 +217,317 @@ Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(31) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(36) FilterExecTransformer +(37) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(37) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(38) ProjectExecTransformer +(39) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(40) ColumnarBroadcastExchange +(41) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(41) BroadcastQueryStage +(42) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(42) InputAdapter +(43) InputAdapter Input [1]: [ps_suppkey#X] -(43) InputIteratorTransformer +(44) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(47) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(48) ProjectExecTransformer +(49) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(50) ColumnarBroadcastExchange +(51) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastQueryStage +(52) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(52) InputAdapter +(53) InputAdapter Input [1]: [n_nationkey#X] -(53) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [n_nationkey#X] -(54) BroadcastHashJoinExecTransformer +(55) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(58) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(59) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(61) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(63) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(65) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(66) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(72) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(73) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(75) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(76) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(77) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(78) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(81) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(82) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(83) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(84) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(86) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(87) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(89) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(92) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(93) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(95) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(96) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(97) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt index f81832ea4c1d..5aef62e33765 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -305,195 +306,199 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt index 5dac875505ea..fbda7224d642 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt @@ -1,37 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- VeloxAppendBatches (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin LeftAnti BuildRight (33) + :- Filter (30) + : +- Scan parquet (29) + +- BroadcastExchange (32) + +- Scan parquet (31) (1) Scan parquet @@ -94,247 +96,261 @@ Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(21) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(22) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(24) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(26) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(29) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(30) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(31) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(32) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(34) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(36) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(37) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(38) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(39) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) -(39) Scan parquet +(41) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(41) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(46) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(47) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(49) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(54) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(55) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(56) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(57) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(58) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- VeloxAppendBatches (46) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ FilterExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index 7ee3b1a0cfd4..6d518ac27214 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -161,133 +162,137 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt index 65a268b41109..bb6c149c39e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (46) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (31) + +- ^ SortExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ InputIteratorTransformer (21) + +- ShuffleQueryStage (19), Statistics(X) + +- ColumnarExchange (18) + +- VeloxAppendBatches (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (45) + +- Exchange (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- BroadcastHashJoin LeftSemi BuildRight (39) + :- Project (34) + : +- Filter (33) + : +- Scan parquet (32) + +- BroadcastExchange (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -113,126 +115,134 @@ Input [2]: [o_orderpriority#X, count#X] Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(17) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(18) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(20) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(24) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(25) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(29) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(32) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(33) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(34) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(35) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(36) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(37) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(38) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(39) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(40) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(41) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(42) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(43) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(45) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(46) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt index a86ee299c7c3..afb9aa369966 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt @@ -1,91 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (67) + +- ^ SortExecTransformer (65) + +- ^ InputIteratorTransformer (64) + +- ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- VeloxAppendBatches (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- VeloxAppendBatches (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (101) + +- Exchange (100) + +- HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Project (85) + : : +- BroadcastHashJoin Inner BuildRight (84) + : : :- Project (80) + : : : +- BroadcastHashJoin Inner BuildRight (79) + : : : :- Project (75) + : : : : +- BroadcastHashJoin Inner BuildLeft (74) + : : : : :- BroadcastExchange (70) + : : : : : +- Filter (69) + : : : : : +- Scan parquet (68) + : : : : +- Project (73) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (78) + : : : +- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) (1) Scan parquet @@ -317,226 +319,234 @@ Input [3]: [n_name#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(53) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(54) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(55) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(56) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(57) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(60) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(61) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(62) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(63) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(64) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(65) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(75) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(76) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(78) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(80) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(85) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(86) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(87) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(88) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(90) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(91) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(93) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(94) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(96) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(97) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(98) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(99) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(100) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(101) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt index 12d6c3ea85e4..ddc921e22d0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt index 571b6c13b6fd..5ca9bbe39ef2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt @@ -1,85 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (62) + +- ^ SortExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ColumnarExchange (56) + +- VeloxAppendBatches (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (94) + +- Exchange (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- BroadcastHashJoin Inner BuildRight (88) + :- Project (84) + : +- BroadcastHashJoin Inner BuildRight (83) + : :- Project (79) + : : +- BroadcastHashJoin Inner BuildRight (78) + : : :- Project (74) + : : : +- BroadcastHashJoin Inner BuildRight (73) + : : : :- Project (69) + : : : : +- BroadcastHashJoin Inner BuildLeft (68) + : : : : :- BroadcastExchange (65) + : : : : : +- Filter (64) + : : : : : +- Scan parquet (63) + : : : : +- Filter (67) + : : : : +- Scan parquet (66) + : : : +- BroadcastExchange (72) + : : : +- Filter (71) + : : : +- Scan parquet (70) + : : +- BroadcastExchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- BroadcastExchange (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -287,218 +289,226 @@ Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(48) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(49) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(55) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(56) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(57) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(58) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(59) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(60) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(63) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(64) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(65) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(66) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(67) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(69) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(72) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(74) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(79) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(81) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(82) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(84) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(89) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(90) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(91) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(92) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(93) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(94) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt index af98ad739c21..6c2c8eb46bc9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt @@ -1,116 +1,118 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (131) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- VeloxAppendBatches (79) + +- ^ ProjectExecTransformer (77) + +- ^ RegularHashAggregateExecTransformer (76) + +- ^ InputIteratorTransformer (75) + +- ShuffleQueryStage (73), Statistics(X) + +- ColumnarExchange (72) + +- VeloxAppendBatches (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (130) + +- Exchange (129) + +- HashAggregate (128) + +- Exchange (127) + +- HashAggregate (126) + +- Project (125) + +- BroadcastHashJoin Inner BuildRight (124) + :- Project (119) + : +- BroadcastHashJoin Inner BuildRight (118) + : :- Project (114) + : : +- BroadcastHashJoin Inner BuildRight (113) + : : :- Project (109) + : : : +- BroadcastHashJoin Inner BuildRight (108) + : : : :- Project (104) + : : : : +- BroadcastHashJoin Inner BuildRight (103) + : : : : :- Project (99) + : : : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : : : :- Project (94) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (93) + : : : : : : :- BroadcastExchange (90) + : : : : : : : +- Project (89) + : : : : : : : +- Filter (88) + : : : : : : : +- Scan parquet (87) + : : : : : : +- Filter (92) + : : : : : : +- Scan parquet (91) + : : : : : +- BroadcastExchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- BroadcastExchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- BroadcastExchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- BroadcastExchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- BroadcastExchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- BroadcastExchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) (1) Scan parquet @@ -420,280 +422,288 @@ Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(71) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(72) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(73) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(74) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(75) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(76) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(77) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(79) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(80) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(82) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(84) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(91) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(92) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(94) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(95) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(96) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(97) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(99) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(100) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(101) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(102) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(105) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(107) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(111) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(112) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(113) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(114) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(117) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(118) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(119) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(120) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(121) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(122) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(123) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(124) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(125) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(126) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(127) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(128) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(129) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(130) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(131) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt index c88eba3bb12e..6a4faab249c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt @@ -1,89 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (99) + +- Exchange (98) + +- HashAggregate (97) + +- Exchange (96) + +- HashAggregate (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (70) + : : : : : +- Project (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Filter (72) + : : : : +- Scan parquet (71) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -311,222 +313,230 @@ Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(59) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(60) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(61) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(62) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(63) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(64) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(67) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(68) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(69) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(70) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(71) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(74) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(75) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(77) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(79) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(80) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(81) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(84) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(85) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(86) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(87) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(89) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(92) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(94) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(95) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(96) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(97) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(98) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(99) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 9316f6f8ebb5..53edb933c1fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index a09e6167ffd3..85176d8c6011 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,78 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -94,358 +101,386 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 47b241e9a343..7c749c0a5ec6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,65 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -81,292 +87,316 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(49) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index 9adf1b6060ef..5cf27c6e0cb9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,46 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -62,196 +66,212 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index fb354643ede9..d3904d8d079e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,49 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -60,223 +64,239 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index 62a2d4a7b617..00b5fb4142f3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 791ededdabda..eab0e2908a10 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,44 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (47) + +- Exchange (46) + +- Project (45) + +- ShuffledHashJoin Inner BuildLeft (44) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Filter (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -60,182 +63,194 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(31) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(34) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(35) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(36) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(37) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(38) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(39) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(41) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(42) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(43) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(45) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(46) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(47) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 2060266a2550..354bd4f3fabd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,59 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -75,270 +80,290 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 504ee433e9e8..848d4e2ce4f8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -72,250 +75,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 78efb8c67470..3d4743403809 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,89 +1,95 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -105,420 +111,444 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(34) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(42) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(48) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(51) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index aa13d1509c29..6ec9ae965ee9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 959bcd4ef703..24be4842e1b8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (96) + +- ^ SortExecTransformer (94) + +- ^ InputIteratorTransformer (93) + +- ShuffleQueryStage (91) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (135) + +- Exchange (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin LeftSemi BuildRight (125) + : :- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin Inner BuildLeft (122) + : :- Exchange (108) + : : +- ShuffledHashJoin LeftSemi BuildRight (107) + : : :- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (121) + : +- Filter (120) + : +- HashAggregate (119) + : +- HashAggregate (118) + : +- ShuffledHashJoin LeftSemi BuildRight (117) + : :- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Exchange (116) + : +- Project (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (131) + +- Project (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,518 +136,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(92) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(93) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(94) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(95) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(96) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(97) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(99) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(101) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(102) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(109) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(110) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(111) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(112) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(113) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(114) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(115) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(116) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(118) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(119) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(120) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(121) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(123) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(124) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(126) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(127) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(130) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(131) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index e9418fc2a71c..b8c363fce329 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,104 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (92) + +- TakeOrderedAndProjectExecTransformer (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildLeft (109) + : : :- Exchange (95) + : : : +- Filter (94) + : : : +- Scan parquet (93) + : : +- Exchange (108) + : : +- ShuffledHashJoin LeftAnti BuildRight (107) + : : :- ShuffledHashJoin LeftSemi BuildRight (102) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Exchange (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (115) + : +- Project (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -120,494 +130,534 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(91) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(92) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(93) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(94) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(95) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(101) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(103) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(104) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(105) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(110) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(111) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(112) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(113) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(114) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(115) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(117) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(118) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(124) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(125) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(126) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(127) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(128) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 2b0fcd16aadc..f336a73676ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,43 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -59,182 +63,198 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index ed7b3ea7d377..f188fa96b0d8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 07a00316284a..42a8fef3563f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,47 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,200 +67,216 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index 766f9f5a1314..378085655899 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,115 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -131,552 +143,600 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index 7aae3dccfd9b..b6f876d48e5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 67c25166ae14..a7054770a17e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,110 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -126,524 +137,568 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 4c1fb04b8660..cdede8445908 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,150 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -166,732 +182,796 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(134) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 582d323dccb2..11a02d0a54d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,114 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -130,548 +142,596 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index 63b30bb5d26b..afea15af53d2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 6213f61ffdf4..5a74265ab590 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,78 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -94,358 +101,386 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 80dbdb2c50ad..8d17beb8c0a9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,65 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -81,549 +87,583 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(49) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ NoopFilter (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) NoopFilter +(80) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(75) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(83) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(95) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index 58a8788ca252..dd1259eb8876 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,46 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -62,196 +66,212 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 8837c6ef4143..d43ad2a9c271 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,49 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -60,223 +64,239 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 2c06906d4fb8..cb3ddbb3a2f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 270cea0f6f46..60521d6c62cd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -59,328 +62,345 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(43) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(47) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index 90c77f0de9b3..029516a40506 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,59 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -75,270 +80,290 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 16321ffa4e59..e2d1503799a9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -72,250 +75,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index fa441bac82cc..a0e052432bb3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,89 +1,95 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -105,420 +111,444 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(34) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(42) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(48) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(51) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index d9ceedf0db6c..440383aa1cd7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index 305f98339a44..d3cff30e0ce2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,508 +135,548 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index 04cc1f47a3d3..bd77a7f7f043 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,103 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -119,490 +129,530 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 9a513e223197..7cf55b4c0f2d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,43 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -59,296 +63,317 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ NoopFilter (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(47) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) NoopFilter +(52) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(49) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 0b11cfa1f763..8b1f048c7d6e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 2eb1f1044104..1b680584826d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,47 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,200 +67,216 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 85b46c76a499..67159dbb648a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,115 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -131,552 +143,600 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index e03830992c2e..fa9c936a0ca1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index a2e4435d7188..71742ea423b5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,110 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -126,524 +137,568 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 47be7c7230dd..319e6c9f1b21 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,150 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -166,732 +182,796 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(134) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index 93001ea6c78d..40dee1752399 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,114 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -130,548 +142,596 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index c254ec8c82ca..545f2e7e086d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 43930a2eb5db..94e1100ea37d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,78 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -94,364 +101,392 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index e8e0cb750c84..41d23099319f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,65 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -81,557 +87,591 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(49) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ NoopFilter (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) NoopFilter +(80) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(75) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(83) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(95) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index ea49c8c6402c..9995164f4c49 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,46 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -62,198 +66,214 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index a7a55ece84ee..53801198fb49 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,49 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -60,225 +64,241 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 412a680dae78..4080a469e711 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,146 +54,154 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index c66afda3aee4..a177fe4bcaec 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -59,330 +62,347 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(43) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(47) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 7bb61df21db4..89a647ffce45 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,59 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -75,273 +80,293 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index c51280b11b0f..42fc32b0bce1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -72,254 +75,266 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 0f3bc73eb825..ca913b2d4c84 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,89 +1,95 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -105,428 +111,452 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(34) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(42) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(48) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(51) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 1d1169055f32..91187ac8d5a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,142 +53,150 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 71c30b25fb2b..01ab88ee0b2b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,518 +135,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index d9d79a967f03..317740080d7b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,103 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -119,500 +129,540 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index feeda6c9fded..0d779c9160cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,43 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -59,298 +63,319 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ NoopFilter (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(47) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) NoopFilter +(52) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(49) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index ee94233a0a5b..8c671a61c9f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,248 +76,264 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 141bd8aa73fd..3d145f0c3bb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,47 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,202 +67,218 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 95a5f3ee722f..08e655f5aa81 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,115 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -131,562 +143,610 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 5987a808f5fd..64624c791f72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 994d1e163a36..71ade94be21d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,110 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -126,534 +137,578 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index b483283286d4..ddeab25c4569 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,150 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -166,746 +182,810 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(134) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 5d1350564eac..634e3516a710 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,114 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -130,558 +142,606 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt index 656f98574483..63b7d317f3cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 313572f951ad..c5fcd91867cb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -1,78 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -94,358 +101,386 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index e91064e3580f..59de06707aad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -1,65 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -81,292 +87,316 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(49) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index 8bad82473a58..a8ac5d0d2c1b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -1,46 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -62,196 +66,212 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index 3268390701fc..d65867ecf822 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -1,49 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -60,223 +64,239 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt index dd4a53cfa822..2bc0be8fcb67 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index c456274bfeaa..0d21930825c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -1,44 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (47) + +- Exchange (46) + +- Project (45) + +- ShuffledHashJoin Inner BuildLeft (44) + :- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Filter (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -60,182 +63,194 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(31) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(34) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(35) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(36) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(37) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(38) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(39) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(41) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(42) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(43) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(45) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(46) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(47) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index 7bbe4249fdcd..cd3e53ad7bd6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -1,59 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -75,270 +80,290 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index 248f638db9ad..fc17c87d7df0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -72,250 +75,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index 428186218984..fc65f4b52897 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -1,89 +1,95 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -105,420 +111,444 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(34) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(42) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(51) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt index 4ea9d09e13f7..d3d74c5ba792 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt index 4fe956080086..df1ae98f903e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (96) + +- ^ SortExecTransformer (94) + +- ^ InputIteratorTransformer (93) + +- ShuffleQueryStage (91) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (135) + +- Exchange (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin LeftSemi BuildRight (125) + : :- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin Inner BuildLeft (122) + : :- Exchange (108) + : : +- ShuffledHashJoin LeftSemi BuildRight (107) + : : :- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (121) + : +- Filter (120) + : +- HashAggregate (119) + : +- HashAggregate (118) + : +- ShuffledHashJoin LeftSemi BuildRight (117) + : :- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Exchange (116) + : +- Project (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (131) + +- Project (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,518 +136,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(92) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(93) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(94) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(95) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(96) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(97) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(99) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(101) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(102) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(109) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(110) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(111) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(112) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(113) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(114) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(115) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(116) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(118) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(119) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(120) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(121) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(123) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(124) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(126) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(127) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(130) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(131) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index 6f94bd5c8d01..c0f3602f9fe6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -1,104 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (129) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (92) + +- TakeOrderedAndProjectExecTransformer (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (111) + : : +- Project (110) + : : +- ShuffledHashJoin Inner BuildLeft (109) + : : :- Exchange (95) + : : : +- Filter (94) + : : : +- Scan parquet (93) + : : +- Exchange (108) + : : +- ShuffledHashJoin LeftAnti BuildRight (107) + : : :- ShuffledHashJoin LeftSemi BuildRight (102) + : : : :- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Exchange (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (115) + : +- Project (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -120,494 +130,534 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(91) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(92) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(93) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(94) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(95) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(101) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(103) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(104) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(105) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(109) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(110) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(111) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(112) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(113) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(114) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(115) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(117) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(118) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(124) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(125) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(126) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(127) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(128) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(129) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index 77c3b584125b..5ab0811e658f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -1,43 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -59,182 +63,198 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index 6e4f9178de19..c51701bd0840 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index d2e7a9cffbd6..1b95ae3dbf39 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -1,47 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,200 +67,216 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index aff82bdec961..c31fbccc1e59 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -1,115 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -131,552 +143,600 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt index b39d8c1b2aec..786a89fe715a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index f0650d65e2cf..06b84fdca2c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -1,110 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -126,524 +137,568 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index e2a1907549f0..e9fdc420f128 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -1,150 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -166,732 +182,796 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(134) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index b86e86d560e5..d6bc308a9c2a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -1,114 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -130,548 +142,596 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt index 89de3133895b..22dd5100c4fb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index 680971de4c1d..fcb13291c838 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -1,78 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -94,358 +101,386 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index b964ec1ed8d4..bbd20320b798 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -1,65 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -81,549 +87,583 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(49) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ FilterExecTransformer (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer +(80) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(75) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(83) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(95) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index fabde3ecd687..194b60bb7713 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -1,46 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -62,196 +66,212 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index 6ca6e75ac545..a9d4e199cfd2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -1,49 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -60,223 +64,239 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt index c1a4a2fe884f..afac58cb52bc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index a45c61431782..fec017400c11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -59,328 +62,345 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(43) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(47) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index 9a927b77805e..15dd2fa6da8e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -1,59 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -75,270 +80,290 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 48ba7ddab093..69af0fd38e92 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -72,250 +75,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index a15a185bbd77..8db05ed7572c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -1,89 +1,95 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -105,420 +111,444 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(34) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(42) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(51) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt index 80f0189d5d65..14a5515d1e79 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt index a0dcc7029a35..7b840720bc90 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,508 +135,548 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index 3635363cfe47..5c05ec24757e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -1,103 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -119,490 +129,530 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index 3c18ab436ed2..af5c086c274a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -1,43 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -59,316 +63,338 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(47) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(49) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index eebd274d4709..51408f03e4e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index f05c9a5378c6..4da32d7f70ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -1,47 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,200 +67,216 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 027ea4a926d2..2669a9fce3ae 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -1,115 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -131,552 +143,600 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt index 68854bdea473..3432579a0de0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index eb5979fb5d84..b5abf7e36164 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -1,110 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -126,524 +137,568 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index 98bba133502c..47886e292bf7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -1,150 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -166,732 +182,796 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(134) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 7f3917f80457..0bad5c20cf05 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -1,114 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -130,548 +142,596 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt index 090a9522f13a..1e53cd90e1b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt @@ -1,29 +1,31 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (30) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (21) + +- ^ SortExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (29) + +- Exchange (28) + +- HashAggregate (27) + +- Exchange (26) + +- HashAggregate (25) + +- Project (24) + +- Filter (23) + +- Scan parquet (22) (1) Scan parquet @@ -56,97 +58,105 @@ Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] +Arguments: X + +(8) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(15) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(19) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(22) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(23) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(24) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(25) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(26) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(27) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(28) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(29) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(30) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 02cf374dd013..5be72ee42483 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -1,78 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (67) + +- TakeOrderedAndProjectExecTransformer (66) + +- ^ ProjectExecTransformer (64) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- Exchange (91) + +- HashAggregate (90) + +- Project (89) + +- ShuffledHashJoin Inner BuildRight (88) + :- Exchange (84) + : +- Project (83) + : +- ShuffledHashJoin Inner BuildRight (82) + : :- Exchange (77) + : : +- Project (76) + : : +- ShuffledHashJoin Inner BuildRight (75) + : : :- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Exchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (81) + : +- Project (80) + : +- Filter (79) + : +- Scan parquet (78) + +- Exchange (87) + +- Filter (86) + +- Scan parquet (85) (1) Scan parquet @@ -94,364 +101,392 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(59) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(60) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(61) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(62) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(66) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(67) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(68) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(69) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(70) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(71) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(72) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(73) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(74) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(76) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(77) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(78) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(79) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(80) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(81) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(85) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(86) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(87) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(89) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(90) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(91) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(92) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index b14866370aab..d9dbbfe0dbe9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -1,65 +1,71 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (78) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (56) + +- ^ SortExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- VeloxAppendBatches (49) + +- ^ FilterExecTransformer (47) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- VeloxAppendBatches (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (77) + +- Exchange (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Project (71) + +- ShuffledHashJoin Inner BuildRight (70) + :- Exchange (65) + : +- Project (64) + : +- ShuffledHashJoin Inner BuildRight (63) + : :- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Exchange (62) + : +- Filter (61) + : +- Scan parquet (60) + +- Exchange (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) (1) Scan parquet @@ -81,557 +87,591 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] +Arguments: X + +(42) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(43) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(44) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(45) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(49) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(50) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(52) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(54) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(57) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(58) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(59) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(60) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(61) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(62) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(63) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(64) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(65) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(66) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(67) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(68) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(69) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(70) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(71) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(73) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(74) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(75) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(76) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(77) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(78) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (110) + +- ^ ProjectExecTransformer (108) + +- ^ RegularHashAggregateExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ ProjectExecTransformer (105) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + :- ^ InputIteratorTransformer (99) + : +- ShuffleQueryStage (97), Statistics(X) + : +- ColumnarExchange (96) + : +- VeloxAppendBatches (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (87) + : : +- ShuffleQueryStage (85), Statistics(X) + : : +- ColumnarExchange (84) + : : +- VeloxAppendBatches (83) + : : +- ^ ProjectExecTransformer (81) + : : +- ^ FilterExecTransformer (80) + : : +- ^ Scan parquet (79) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ReusedExchange (88) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ReusedExchange (100) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- ShuffledHashJoin Inner BuildRight (124) + :- Exchange (119) + : +- Project (118) + : +- ShuffledHashJoin Inner BuildRight (117) + : :- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Exchange (123) + +- Project (122) + +- Filter (121) + +- Scan parquet (120) + + +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer +(80) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(75) ProjectExecTransformer +(81) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(83) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(86) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(88) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(90) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(93) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(95) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(96) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(97) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(98) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(99) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(101) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(102) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(103) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(105) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(107) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(108) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(109) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(110) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(111) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(112) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(113) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(114) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(115) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(116) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(118) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(119) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(120) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(121) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(122) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(123) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(124) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 27765da815dd..63c356d6d1bf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -1,46 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin Inner BuildLeft (46) + :- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -62,198 +66,214 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] +Arguments: X + +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(39) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(40) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(42) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(43) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(44) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(45) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(47) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(48) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(49) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(50) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(51) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(52) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index 79b9dcd18bd1..812a7be868b6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -1,49 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftOuter BuildRight (47) + :- Exchange (42) + : +- Scan parquet (41) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -60,225 +64,241 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_count#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(34) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(38) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(41) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(45) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(46) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(48) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(50) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(51) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(52) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(53) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(54) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(55) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt index 4db67eada562..c6f425f00868 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,146 +54,154 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index c8f4e2c84ac3..e30eec3d854f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- VeloxAppendBatches (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (44) + +- Exchange (43) + +- Project (42) + +- ShuffledHashJoin Inner BuildLeft (41) + :- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -59,330 +62,347 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(17) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(18) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(19) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(20) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(23) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(27) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(29) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (71) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (62) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ ProjectExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) -(43) Scan parquet +(46) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(47) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(52) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] +Arguments: X + +(53) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(55) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(58) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(62) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(63) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(64) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(65) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(67) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(69) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(70) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index 247b853fb1c4..eff7577281e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -1,59 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- VeloxAppendBatches (40) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- VeloxAppendBatches (33) + +- ^ ProjectExecTransformer (31) + +- ^ FlushableHashAggregateExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (68) + +- Exchange (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- Project (60) + +- ShuffledHashJoin Inner BuildRight (59) + :- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -75,273 +80,293 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] +Arguments: X + +(25) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(45) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(48) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(49) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(50) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(51) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(52) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(53) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(54) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(55) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(58) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(59) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(60) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(61) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(68) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index fad48184fed1..649bfcbe40e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -1,56 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ RegularHashAggregateExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ ProjectExecTransformer (35) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- ShuffledHashJoin Inner BuildRight (56) + :- Project (49) + : +- ShuffledHashJoin Inner BuildRight (48) + : :- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Filter (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -72,254 +75,266 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] +Arguments: X + +(27) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(29) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(33) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(34) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(38) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(41) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(42) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(43) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(44) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(45) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(46) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(47) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(49) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(50) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(51) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(52) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(53) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(54) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(55) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(56) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index 5a6f04064349..c3075e511782 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -1,89 +1,95 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- TakeOrderedAndProjectExecTransformer (69) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ ProjectExecTransformer (65) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (64) + :- ^ InputIteratorTransformer (46) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- VeloxAppendBatches (42) + : +- ^ ProjectExecTransformer (40) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- VeloxAppendBatches (34) + : +- ^ ProjectExecTransformer (32) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (30) + : +- ^ FilterExecTransformer (29) + : +- ^ RegularHashAggregateExecTransformer (28) + : +- ^ InputIteratorTransformer (27) + : +- ShuffleQueryStage (25), Statistics(X) + : +- ColumnarExchange (24) + : +- VeloxAppendBatches (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (63) + :- ^ InputIteratorTransformer (55) + : +- ShuffleQueryStage (53), Statistics(X) + : +- ColumnarExchange (52) + : +- VeloxAppendBatches (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ ProjectExecTransformer (62) + +- ^ FilterExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ InputIteratorTransformer (59) + +- ShuffleQueryStage (57), Statistics(X) + +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -105,428 +111,452 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(29) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(32) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(34) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(35) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(37) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(39) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(42) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(43) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(45) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(47) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(48) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(49) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(51) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(52) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(54) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(55) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(56) ReusedExchange [Reuses operator id: 24] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(57) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(58) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(59) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(61) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(64) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(65) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(68) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(69) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt index 2f7bb7995dd9..baf4b2a51607 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,142 +53,150 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt index e8aa97a29e7a..7ddecfe855eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,518 +135,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index 323db548d24e..7b8c173fc086 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -1,103 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- ^ RegularHashAggregateExecTransformer (89) + +- ^ InputIteratorTransformer (88) + +- ShuffleQueryStage (86), Statistics(X) + +- ColumnarExchange (85) + +- VeloxAppendBatches (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -119,500 +129,540 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(85) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(87) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(88) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(89) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(90) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index add978887b0a..d6ec93a97fc6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -1,43 +1,47 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- ShuffledHashJoin LeftAnti BuildRight (43) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (42) + +- Scan parquet (41) (1) Scan parquet @@ -59,318 +63,340 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(33) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(35) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(38) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(39) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(40) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(41) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(42) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(44) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(45) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(46) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(47) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(48) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(49) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) -(47) Scan parquet +(51) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(49) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(54) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(56) VeloxAppendBatches +Input [2]: [sum#X, count#X] +Arguments: X + +(57) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(58) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(59) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(63) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(64) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(65) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(66) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(67) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(68) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(69) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(70) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (70) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ InputIteratorTransformer (60) + +- ShuffleQueryStage (58), Statistics(X) + +- ColumnarExchange (57) + +- VeloxAppendBatches (56) + +- ^ FlushableHashAggregateExecTransformer (54) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file + HashAggregate (69) + +- Exchange (68) + +- HashAggregate (67) + +- Project (66) + +- Filter (65) + +- Scan parquet (64) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index 69dc65c58e21..709a1700b5c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,248 +76,264 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index d7f42dd7b351..a82dbf288086 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -1,47 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (38) + +- ^ SortExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- VeloxAppendBatches (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- ShuffledHashJoin LeftSemi BuildRight (47) + :- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,202 +67,218 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] +Arguments: X + +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(27) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(29) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(36) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(38) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(40) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(41) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(42) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(43) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(44) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(45) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(46) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(48) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(49) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(51) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(53) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index 129b02c9548e..f20a52b91a8f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -1,115 +1,127 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (146) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (145) + +- Exchange (144) + +- HashAggregate (143) + +- Exchange (142) + +- HashAggregate (141) + +- Project (140) + +- ShuffledHashJoin Inner BuildRight (139) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Project (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (138) + +- Project (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -131,562 +143,610 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(100) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(109) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(110) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(111) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(112) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(115) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(116) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(119) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(121) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(122) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(125) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(127) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(128) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(131) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(133) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(134) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(137) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(138) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(139) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(140) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(141) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(142) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(143) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(144) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(145) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt index 12d6c3ea85e4..ddc921e22d0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index 1e6af0683a39..710c6f3ba189 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -1,110 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (139) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (101) + +- ^ SortExecTransformer (99) + +- ^ InputIteratorTransformer (98) + +- ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- VeloxAppendBatches (94) + +- ^ RegularHashAggregateExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- VeloxAppendBatches (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (138) + +- Exchange (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (128) + : +- Project (127) + : +- ShuffledHashJoin Inner BuildRight (126) + : :- Exchange (122) + : : +- Project (121) + : : +- ShuffledHashJoin Inner BuildRight (120) + : : :- Exchange (116) + : : : +- Project (115) + : : : +- ShuffledHashJoin Inner BuildRight (114) + : : : :- Exchange (110) + : : : : +- Project (109) + : : : : +- ShuffledHashJoin Inner BuildLeft (108) + : : : : :- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Exchange (107) + : : : : +- Filter (106) + : : : : +- Scan parquet (105) + : : : +- Exchange (113) + : : : +- Filter (112) + : : : +- Scan parquet (111) + : : +- Exchange (119) + : : +- Filter (118) + : : +- Scan parquet (117) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) (1) Scan parquet @@ -126,534 +137,578 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) VeloxAppendBatches +Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] +Arguments: X + +(88) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(90) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(92) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(96) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(97) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(98) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(99) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(100) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(101) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(105) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(106) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(107) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(109) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(110) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(112) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(115) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(116) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(117) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(118) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(119) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(121) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(122) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(123) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(124) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(125) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(127) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(128) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(129) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(130) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(131) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(133) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(134) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(135) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(136) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(137) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(138) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(139) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index cfc03953fe18..953a3a8f0a7c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -1,150 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (193) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (141) + +- ^ SortExecTransformer (139) + +- ^ InputIteratorTransformer (138) + +- ShuffleQueryStage (136), Statistics(X) + +- ColumnarExchange (135) + +- VeloxAppendBatches (134) + +- ^ ProjectExecTransformer (132) + +- ^ RegularHashAggregateExecTransformer (131) + +- ^ InputIteratorTransformer (130) + +- ShuffleQueryStage (128), Statistics(X) + +- ColumnarExchange (127) + +- VeloxAppendBatches (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (192) + +- Exchange (191) + +- HashAggregate (190) + +- Exchange (189) + +- HashAggregate (188) + +- Project (187) + +- ShuffledHashJoin Inner BuildRight (186) + :- Exchange (181) + : +- Project (180) + : +- ShuffledHashJoin Inner BuildRight (179) + : :- Exchange (175) + : : +- Project (174) + : : +- ShuffledHashJoin Inner BuildRight (173) + : : :- Exchange (169) + : : : +- Project (168) + : : : +- ShuffledHashJoin Inner BuildRight (167) + : : : :- Exchange (163) + : : : : +- Project (162) + : : : : +- ShuffledHashJoin Inner BuildRight (161) + : : : : :- Exchange (157) + : : : : : +- Project (156) + : : : : : +- ShuffledHashJoin Inner BuildRight (155) + : : : : : :- Exchange (151) + : : : : : : +- Project (150) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) + : : : : : : :- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Exchange (154) + : : : : : +- Filter (153) + : : : : : +- Scan parquet (152) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Exchange (166) + : : : +- Filter (165) + : : : +- Scan parquet (164) + : : +- Exchange (172) + : : +- Filter (171) + : : +- Scan parquet (170) + : +- Exchange (178) + : +- Filter (177) + : +- Scan parquet (176) + +- Exchange (185) + +- Project (184) + +- Filter (183) + +- Scan parquet (182) (1) Scan parquet @@ -166,746 +182,810 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) VeloxAppendBatches +Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(127) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(128) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(129) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(130) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(131) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(132) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(133) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(134) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(135) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(136) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(137) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(138) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(139) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(140) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(141) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(142) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(143) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(144) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(145) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(146) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(147) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(148) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(150) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(151) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(152) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(153) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(154) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(156) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(157) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(158) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(159) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(160) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(161) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(162) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(163) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(164) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(165) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(166) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(167) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(170) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(171) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(172) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(173) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(174) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(175) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(176) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(177) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(178) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(179) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(180) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(181) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(182) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(183) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(184) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(185) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(186) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(187) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(188) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(189) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(190) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(191) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(192) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(193) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index 9a8c9a87aac2..492ff1aeadd0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -1,114 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (106) + +- ^ SortExecTransformer (104) + +- ^ InputIteratorTransformer (103) + +- ShuffleQueryStage (101), Statistics(X) + +- ColumnarExchange (100) + +- VeloxAppendBatches (99) + +- ^ RegularHashAggregateExecTransformer (97) + +- ^ InputIteratorTransformer (96) + +- ShuffleQueryStage (94), Statistics(X) + +- ColumnarExchange (93) + +- VeloxAppendBatches (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (134) + : +- Project (133) + : +- ShuffledHashJoin Inner BuildRight (132) + : :- Exchange (128) + : : +- Project (127) + : : +- ShuffledHashJoin Inner BuildRight (126) + : : :- Exchange (122) + : : : +- Project (121) + : : : +- ShuffledHashJoin Inner BuildRight (120) + : : : :- Exchange (116) + : : : : +- Project (115) + : : : : +- ShuffledHashJoin Inner BuildLeft (114) + : : : : :- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Exchange (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Exchange (119) + : : : +- Filter (118) + : : : +- Scan parquet (117) + : : +- Exchange (125) + : : +- Filter (124) + : : +- Scan parquet (123) + : +- Exchange (131) + : +- Filter (130) + : +- Scan parquet (129) + +- Exchange (137) + +- Filter (136) + +- Scan parquet (135) (1) Scan parquet @@ -130,558 +142,606 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) VeloxAppendBatches +Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] +Arguments: X + +(93) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(94) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(95) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(96) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(97) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(104) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(107) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(108) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(109) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(110) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(111) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(112) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(113) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(115) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(116) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(117) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(118) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(119) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(121) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(122) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(123) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(124) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(125) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(126) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(127) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(128) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(129) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(130) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(131) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(133) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(134) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(135) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(136) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(137) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(139) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(140) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(141) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(142) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(143) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(144) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala index e4356cec8ff1..9e120945be34 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenImplicitsTest.scala @@ -79,10 +79,10 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait { testGluten("fallbackSummary with shuffle") { withAQEEnabledAndDisabled { val df = spark.sql("SELECT c2 FROM t1 group by c2").filter(_.getLong(0) > 0) - assert(df.fallbackSummary().numGlutenNodes == 5, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) df.collect() - assert(df.fallbackSummary().numGlutenNodes == 5, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) } } @@ -119,10 +119,10 @@ class GlutenImplicitsTest extends GlutenSQLTestsBaseTrait { testGluten("fallbackSummary with cached data and shuffle") { withAQEEnabledAndDisabled { val df = spark.sql("select * from t1").filter(_.getLong(0) > 0).cache.repartition() - assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 7, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) df.collect() - assert(df.fallbackSummary().numGlutenNodes == 6, df.fallbackSummary()) + assert(df.fallbackSummary().numGlutenNodes == 7, df.fallbackSummary()) assert(df.fallbackSummary().numFallbackNodes == 1, df.fallbackSummary()) } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala index 8896541c29d2..c25e65cf0b68 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/errors/GlutenQueryExecutionErrorsSuite.scala @@ -16,11 +16,20 @@ */ package org.apache.spark.sql.errors +import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsBaseTrait class GlutenQueryExecutionErrorsSuite extends QueryExecutionErrorsSuite with GlutenSQLTestsBaseTrait { + + override def sparkConf: SparkConf = { + // Disables VeloxAppendBatches in which GeneralOutIterator wraps vanilla Spark's exceptions + // with GlutenException. + super.sparkConf + .set("spark.gluten.sql.columnar.backend.velox.coalesceBatchesBeforeShuffle", "false") + } + override protected def getResourceParquetFilePath(name: String): String = { getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + name } diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index a4e5a4425e3b..eb6118ffc74e 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -28,7 +28,6 @@ import java.util.Locale import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ -import scala.collection.JavaConverters.collectionAsScalaIterableConverter case class GlutenNumaBindingInfo( enableNumaBinding: Boolean, @@ -307,10 +306,12 @@ class GlutenConfig(conf: SQLConf) extends Logging { def veloxCoalesceBatchesBeforeShuffle: Boolean = conf.getConf(COLUMNAR_VELOX_COALESCE_BATCHES_BEFORE_SHUFFLE) - def veloxMinBatchSizeForShuffle: Int = + def veloxMinBatchSizeForShuffle: Int = { + val defaultSize: Int = (0.8 * conf.getConf(COLUMNAR_MAX_BATCH_SIZE)).toInt.max(1) conf .getConf(COLUMNAR_VELOX_MIN_BATCH_SIZE_FOR_SHUFFLE) - .getOrElse(conf.getConf(COLUMNAR_MAX_BATCH_SIZE)) + .getOrElse(defaultSize) + } def chColumnarShufflePreferSpill: Boolean = conf.getConf(COLUMNAR_CH_SHUFFLE_PREFER_SPILL_ENABLED) @@ -1425,9 +1426,9 @@ object GlutenConfig { buildConf("spark.gluten.sql.columnar.backend.velox.coalesceBatchesBeforeShuffle") .internal() .doc(s"If true, combine small columnar batches together before sending to shuffle. " + - s"The default minimum output batch size is equal to $GLUTEN_MAX_BATCH_SIZE_KEY") + s"The default minimum output batch size is equal to 0.8 * $GLUTEN_MAX_BATCH_SIZE_KEY") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COLUMNAR_VELOX_MIN_BATCH_SIZE_FOR_SHUFFLE = buildConf("spark.gluten.sql.columnar.backend.velox.minBatchSizeForShuffle") From 3ba31726d77e3bacd60419253d1c2b2b59c84239 Mon Sep 17 00:00:00 2001 From: JiaKe Date: Tue, 18 Jun 2024 15:50:51 +0800 Subject: [PATCH 290/402] [VL] Daily Update Velox Version (2024_06_18) (#6120) --- ep/build-velox/src/get_velox.sh | 2 +- .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 2 -- .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 2 -- .../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 2 -- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 0e72469a43fc..8d5bd8d167fe 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_17 +VELOX_BRANCH=2024_06_18 VELOX_HOME="" #Set on run gluten on HDFS diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 79357e9e220d..8418cba237e3 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1104,8 +1104,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-31116: Select nested schema with case insensitive mode") // exclude as original metric not correct when task offloaded to velox .exclude("SPARK-37585: test input metrics for DSV2 with output limits") - // Unknown. Need to investigate. - .exclude("SPARK-30362: test input metrics for DSV2") // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type .exclude("File source v2: support passing data filters to FileScan without partitionFilters") // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 060f199d1e23..91127c4ba9bb 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1104,8 +1104,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-31116: Select nested schema with case insensitive mode") // exclude as original metric not correct when task offloaded to velox .exclude("SPARK-37585: test input metrics for DSV2 with output limits") - // TODO(yuan): fix the input bytes on ORC code path - .exclude("SPARK-30362: test input metrics for DSV2") // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type .exclude("File source v2: support passing data filters to FileScan without partitionFilters") // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 2911512f5512..6162b5651980 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1120,8 +1120,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-31116: Select nested schema with case insensitive mode") // exclude as original metric not correct when task offloaded to velox .exclude("SPARK-37585: test input metrics for DSV2 with output limits") - // TODO(yuan): fix the input bytes on ORC code path - .exclude("SPARK-30362: test input metrics for DSV2") // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type .exclude("File source v2: support passing data filters to FileScan without partitionFilters") // DISABLED: GLUTEN-4893 Vanilla UT checks scan operator by exactly matching the class type From 800cadd0f4f71d0ebedb5fbf6428442ae52b77ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Tue, 18 Jun 2024 21:16:52 +0800 Subject: [PATCH 291/402] [VL] Support Spark transform_keys, transform_values function (#6095) --- .../velox/VeloxSparkPlanExecApi.scala | 5 ++- .../ScalarFunctionsValidateSuite.scala | 40 +++++++++++++++++++ .../expression/ExpressionConverter.scala | 15 +++++++ .../gluten/expression/ExpressionNames.scala | 2 + 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 26b4c508221d..ebf82ea767a4 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -22,6 +22,7 @@ import org.apache.gluten.datasource.ArrowConvertorRule import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ +import org.apache.gluten.expression.ExpressionNames.{TRANSFORM_KEYS, TRANSFORM_VALUES} import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} import org.apache.gluten.extension._ import org.apache.gluten.extension.columnar.TransformHints @@ -854,7 +855,9 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { Sig[VeloxCollectList](ExpressionNames.COLLECT_LIST), Sig[VeloxCollectSet](ExpressionNames.COLLECT_SET), Sig[VeloxBloomFilterMightContain](ExpressionNames.MIGHT_CONTAIN), - Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG) + Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG), + Sig[TransformKeys](TRANSFORM_KEYS), + Sig[TransformValues](TRANSFORM_VALUES) ) } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index a23fdf243888..9718b8e7358e 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -537,6 +537,46 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("test transform_keys function") { + withTempPath { + path => + Seq( + Map[String, Int]("a" -> 1, "b" -> 2), + Map[String, Int]("a" -> 2, "b" -> 3), + null + ) + .toDF("m") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("map_tbl") + + runQueryAndCompare("select transform_keys(m, (k, v) -> upper(k)) from map_tbl") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + + test("test transform_values function") { + withTempPath { + path => + Seq( + Map[String, Int]("a" -> 1, "b" -> 2), + Map[String, Int]("a" -> 2, "b" -> 3), + null + ) + .toDF("m") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("map_tbl") + + runQueryAndCompare("select transform_values(m, (k, v) -> v + 1) from map_tbl") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + } + test("zip_with") { withTempPath { path => diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 464bbbfd002c..b7b0889dc1eb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -656,6 +656,21 @@ object ExpressionConverter extends SQLConfHelper with Logging { Seq(replaceWithExpressionTransformerInternal(c.child, attributeSeq, expressionsMap)), c ) + case t: TransformKeys => + // default is `EXCEPTION` + val mapKeyDedupPolicy = SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY) + if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) { + // TODO: Remove after fix ready for + // https://github.com/facebookincubator/velox/issues/10219 + throw new GlutenNotSupportException( + "LAST_WIN policy is not supported yet in native to deduplicate map keys" + ) + } + GenericExpressionTransformer( + substraitExprName, + t.children.map(replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), + t + ) case expr => GenericExpressionTransformer( substraitExprName, diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 20db380180e3..112fa677d2cd 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -276,6 +276,8 @@ object ExpressionNames { final val MAP_FROM_ARRAYS = "map_from_arrays" final val MAP_ENTRIES = "map_entries" final val MAP_ZIP_WITH = "map_zip_with" + final val TRANSFORM_KEYS = "transform_keys" + final val TRANSFORM_VALUES = "transform_values" final val STR_TO_MAP = "str_to_map" // struct functions From cf3a98e17ba48bd4fb40343990f5221ea399c82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Wed, 19 Jun 2024 09:33:41 +0800 Subject: [PATCH 292/402] [VL] [Core] Spark Input_file_name Support (#6021) --- .../backendsapi/velox/VeloxIteratorApi.scala | 10 +- .../velox/VeloxSparkPlanExecApi.scala | 3 +- .../extension/InputFileNameReplaceRule.scala | 155 ++++++++++++++++++ .../ScalarFunctionsValidateSuite.scala | 7 + .../columnar/heuristic/HeuristicApplier.scala | 2 +- 5 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 459a7886ea23..880e1e56b852 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -19,6 +19,7 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.GlutenNumaBindingInfo import org.apache.gluten.backendsapi.IteratorApi import org.apache.gluten.execution._ +import org.apache.gluten.extension.InputFileNameReplaceRule import org.apache.gluten.metrics.IMetrics import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.plan.PlanNode @@ -112,7 +113,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { val fileSizes = new JArrayList[JLong]() val modificationTimes = new JArrayList[JLong]() val partitionColumns = new JArrayList[JMap[String, String]] - var metadataColumns = new JArrayList[JMap[String, String]] + val metadataColumns = new JArrayList[JMap[String, String]] files.foreach { file => // The "file.filePath" in PartitionedFile is not the original encoded path, so the decoded @@ -132,6 +133,13 @@ class VeloxIteratorApi extends IteratorApi with Logging { } val metadataColumn = SparkShimLoader.getSparkShims.generateMetadataColumns(file, metadataColumnNames) + metadataColumn.put(InputFileNameReplaceRule.replacedInputFileName, file.filePath.toString) + metadataColumn.put( + InputFileNameReplaceRule.replacedInputFileBlockStart, + file.start.toString) + metadataColumn.put( + InputFileNameReplaceRule.replacedInputFileBlockLength, + file.length.toString) metadataColumns.add(metadataColumn) val partitionColumn = new JHashMap[String, String]() for (i <- 0 until file.partitionValues.numFields) { diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index ebf82ea767a4..71930d7e0f47 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -807,7 +807,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { */ override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = List( BloomFilterMightContainJointRewriteRule.apply, - ArrowScanReplaceRule.apply + ArrowScanReplaceRule.apply, + InputFileNameReplaceRule.apply ) /** diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala new file mode 100644 index 000000000000..cd3f50d8e77f --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, NamedExpression} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{FileSourceScanExec, ProjectExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan +import org.apache.spark.sql.types.{LongType, StringType} + +object InputFileNameReplaceRule { + val replacedInputFileName = "$input_file_name$" + val replacedInputFileBlockStart = "$input_file_block_start$" + val replacedInputFileBlockLength = "$input_file_block_length$" +} + +case class InputFileNameReplaceRule(spark: SparkSession) extends Rule[SparkPlan] { + import InputFileNameReplaceRule._ + + private def isInputFileName(expr: Expression): Boolean = { + expr match { + case _: InputFileName => true + case _ => false + } + } + + private def isInputFileBlockStart(expr: Expression): Boolean = { + expr match { + case _: InputFileBlockStart => true + case _ => false + } + } + + private def isInputFileBlockLength(expr: Expression): Boolean = { + expr match { + case _: InputFileBlockLength => true + case _ => false + } + } + + override def apply(plan: SparkPlan): SparkPlan = { + val replacedExprs = scala.collection.mutable.Map[String, AttributeReference]() + + def hasParquetScan(plan: SparkPlan): Boolean = { + plan match { + case fileScan: FileSourceScanExec + if fileScan.relation.fileFormat.isInstanceOf[ParquetFileFormat] => + true + case batchScan: BatchScanExec => + batchScan.scan match { + case _: ParquetScan => true + case _ => false + } + case _ => plan.children.exists(hasParquetScan) + } + } + + def mayNeedConvert(expr: Expression): Boolean = { + expr match { + case e if isInputFileName(e) => true + case s if isInputFileBlockStart(s) => true + case l if isInputFileBlockLength(l) => true + case other => other.children.exists(mayNeedConvert) + } + } + + def doConvert(expr: Expression): Expression = { + expr match { + case e if isInputFileName(e) => + replacedExprs.getOrElseUpdate( + replacedInputFileName, + AttributeReference(replacedInputFileName, StringType, true)()) + case s if isInputFileBlockStart(s) => + replacedExprs.getOrElseUpdate( + replacedInputFileBlockStart, + AttributeReference(replacedInputFileBlockStart, LongType, true)() + ) + case l if isInputFileBlockLength(l) => + replacedExprs.getOrElseUpdate( + replacedInputFileBlockLength, + AttributeReference(replacedInputFileBlockLength, LongType, true)() + ) + case other => + other.withNewChildren(other.children.map(child => doConvert(child))) + } + } + + def ensureChildOutputHasNewAttrs(plan: SparkPlan): SparkPlan = { + plan match { + case _ @ProjectExec(projectList, child) => + var newProjectList = projectList + for ((_, newAttr) <- replacedExprs) { + if (!newProjectList.exists(attr => attr.exprId == newAttr.exprId)) { + newProjectList = newProjectList :+ newAttr.toAttribute + } + } + val newChild = ensureChildOutputHasNewAttrs(child) + ProjectExec(newProjectList, newChild) + case f: FileSourceScanExec => + var newOutput = f.output + for ((_, newAttr) <- replacedExprs) { + if (!newOutput.exists(attr => attr.exprId == newAttr.exprId)) { + newOutput = newOutput :+ newAttr.toAttribute + } + } + f.copy(output = newOutput) + + case b: BatchScanExec => + var newOutput = b.output + for ((_, newAttr) <- replacedExprs) { + if (!newOutput.exists(attr => attr.exprId == newAttr.exprId)) { + newOutput = newOutput :+ newAttr + } + } + b.copy(output = newOutput) + case other => + val newChildren = other.children.map(ensureChildOutputHasNewAttrs) + other.withNewChildren(newChildren) + } + } + + def replaceInputFileNameInProject(plan: SparkPlan): SparkPlan = { + plan match { + case _ @ProjectExec(projectList, child) + if projectList.exists(mayNeedConvert) && hasParquetScan(plan) => + val newProjectList = projectList.map { + expr => doConvert(expr).asInstanceOf[NamedExpression] + } + val newChild = replaceInputFileNameInProject(ensureChildOutputHasNewAttrs(child)) + ProjectExec(newProjectList, newChild) + case other => + val newChildren = other.children.map(replaceInputFileNameInProject) + other.withNewChildren(newChildren) + } + } + replaceInputFileNameInProject(plan) + } +} diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 9718b8e7358e..d08ba11ee787 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -623,6 +623,13 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("Test input_file_name function") { + runQueryAndCompare("""SELECT input_file_name(), l_orderkey + | from lineitem limit 100""".stripMargin) { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("Test spark_partition_id function") { runQueryAndCompare("""SELECT spark_partition_id(), l_orderkey | from lineitem limit 100""".stripMargin) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index ad68786e6579..d925bc231cd9 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -96,11 +96,11 @@ class HeuristicApplier(session: SparkSession) (spark: SparkSession) => FallbackOnANSIMode(spark), (spark: SparkSession) => FallbackMultiCodegens(spark), (spark: SparkSession) => PlanOneRowRelation(spark), - (_: SparkSession) => FallbackEmptySchemaRelation(), (_: SparkSession) => RewriteSubqueryBroadcast() ) ::: BackendsApiManager.getSparkPlanExecApiInstance.genExtendedColumnarValidationRules() ::: List( + (_: SparkSession) => FallbackEmptySchemaRelation(), (spark: SparkSession) => MergeTwoPhasesHashBaseAggregate(spark), (_: SparkSession) => RewriteSparkPlanRulesManager(), (_: SparkSession) => AddTransformHintRule() From c9a6648f8887bd97af8f547d1091324bd56618b3 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Wed, 19 Jun 2024 10:18:46 +0800 Subject: [PATCH 293/402] [GLUTEN-6078][CH] Enable mergetree hdfs suite (#6080) What changes were proposed in this pull request? (Fixes: #6078) How was this patch tested? Test by ut --- ...nClickHouseMergeTreeWriteOnHDFSSuite.scala | 10 +++---- cpp-ch/clickhouse.version | 2 +- cpp-ch/local-engine/Common/CHUtil.cpp | 8 +++-- .../Disks/ObjectStorages/GlutenDiskHDFS.cpp | 12 +++----- .../Disks/ObjectStorages/GlutenDiskHDFS.h | 29 ++++++++++++++----- .../GlutenHDFSObjectStorage.cpp | 2 +- .../ObjectStorages/GlutenHDFSObjectStorage.h | 4 +-- .../registerGlutenDiskObjectStorage.cpp | 2 +- .../Disks/registerGlutenDisks.cpp | 17 +++++++++-- 9 files changed, 54 insertions(+), 32 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala index ca5b39fff1ac..56b8f056bc25 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala @@ -74,7 +74,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite FileUtils.deleteDirectory(new File(HDFS_METADATA_PATH)) } - ignore("test mergetree table write") { + test("test mergetree table write") { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_hdfs; |""".stripMargin) @@ -157,7 +157,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite spark.sql("drop table lineitem_mergetree_hdfs") } - ignore("test mergetree write with orderby keys / primary keys") { + test("test mergetree write with orderby keys / primary keys") { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_orderbykey_hdfs; |""".stripMargin) @@ -254,7 +254,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite spark.sql("drop table lineitem_mergetree_orderbykey_hdfs") } - ignore("test mergetree write with partition") { + test("test mergetree write with partition") { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_partition_hdfs; |""".stripMargin) @@ -435,7 +435,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite spark.sql("drop table lineitem_mergetree_partition_hdfs") } - ignore("test mergetree write with bucket table") { + test("test mergetree write with bucket table") { spark.sql(s""" |DROP TABLE IF EXISTS lineitem_mergetree_bucket_hdfs; |""".stripMargin) @@ -537,7 +537,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite spark.sql("drop table lineitem_mergetree_bucket_hdfs") } - ignore("test mergetree write with the path based") { + test("test mergetree write with the path based") { val dataPath = s"$HDFS_URL/test/lineitem_mergetree_bucket_hdfs" val sourceDF = spark.sql(s""" diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index e374d3f5fd9e..2bbb2945334b 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence CH_BRANCH=rebase_ch/20240616 -CH_COMMIT=e0e4b947245 +CH_COMMIT=803ee50cdb9fd56a5d77c710da1cbd071a74d1da diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index a4634c3f3bc7..937beae99a6b 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -765,6 +765,11 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) // We must set the application type to CLIENT to avoid ServerUUID::get() throw exception global_context->setApplicationType(Context::ApplicationType::CLIENT); } + else + { + // just for ut + global_context->updateStorageConfiguration(*config); + } } void BackendInitializerUtil::applyGlobalConfigAndSettings(DB::Context::ConfigurationPtr config, DB::Settings & settings) @@ -801,10 +806,7 @@ void registerAllFunctions() void registerGlutenDisks() { registerDisks(true); - -#if USE_AWS_S3 registerGlutenDisks(true); -#endif } void BackendInitializerUtil::registerAllFactories() diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp index cdbe6c72897c..07a7aa6bd006 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp @@ -32,16 +32,11 @@ void GlutenDiskHDFS::createDirectory(const String & path) hdfsCreateDirectory(hdfs_object_storage->getHDFSFS(), path.c_str()); } -String GlutenDiskHDFS::path2AbsPath(const String & path) -{ - return getObjectStorage()->generateObjectKeyForPath(path).serialize(); -} - void GlutenDiskHDFS::createDirectories(const String & path) { DiskObjectStorage::createDirectories(path); - auto* hdfs = hdfs_object_storage->getHDFSFS(); - fs::path p = path; + auto * hdfs = hdfs_object_storage->getHDFSFS(); + fs::path p = "/" + path; std::vector paths_created; while (hdfsExists(hdfs, p.c_str()) < 0) { @@ -69,7 +64,8 @@ DiskObjectStoragePtr GlutenDiskHDFS::createDiskObjectStorage() getMetadataStorage(), getObjectStorage(), SerializedPlanParser::global_context->getConfigRef(), - config_prefix); + config_prefix, + object_storage_creator); } std::unique_ptr GlutenDiskHDFS::writeFile( diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h index 4e375b283951..222b9f8928a3 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h @@ -37,12 +37,15 @@ class GlutenDiskHDFS : public DB::DiskObjectStorage DB::MetadataStoragePtr metadata_storage_, DB::ObjectStoragePtr object_storage_, const Poco::Util::AbstractConfiguration & config, - const String & config_prefix) + const String & config_prefix, + std::function _object_storage_creator) : DiskObjectStorage(name_, object_key_prefix_, metadata_storage_, object_storage_, config, config_prefix) + , object_key_prefix(object_key_prefix_) + , hdfs_config_prefix(config_prefix) + , object_storage_creator(_object_storage_creator) { - chassert(dynamic_cast(object_storage_.get()) != nullptr); - object_key_prefix = object_key_prefix_; - hdfs_object_storage = dynamic_cast(object_storage_.get()); + hdfs_object_storage = typeid_cast>(object_storage_); hdfsSetWorkingDirectory(hdfs_object_storage->getHDFSFS(), "/"); auto max_speed = config.getUInt(config_prefix + ".write_speed", 450); throttler = std::make_shared(max_speed); @@ -59,12 +62,24 @@ class GlutenDiskHDFS : public DB::DiskObjectStorage std::unique_ptr writeFile(const String& path, size_t buf_size, DB::WriteMode mode, const DB::WriteSettings& settings) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + DB::ContextPtr context, + const String & config_prefix, + const DB::DisksMap & map) override + { + DB::ObjectStoragePtr tmp = object_storage_creator(config, context); + hdfs_object_storage = typeid_cast>(tmp); + object_storage = hdfs_object_storage; + } private: - String path2AbsPath(const String & path); - - GlutenHDFSObjectStorage * hdfs_object_storage; + std::shared_ptr hdfs_object_storage; String object_key_prefix; DB::ThrottlerPtr throttler; + const String hdfs_config_prefix; + std::function + object_storage_creator; }; #endif } diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp index 60b82ec845bb..cab87d66d884 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp @@ -38,7 +38,7 @@ DB::ObjectStorageKey local_engine::GlutenHDFSObjectStorage::generateObjectKeyFor initializeHDFSFS(); /// what ever data_source_description.description value is, consider that key as relative key chassert(data_directory.starts_with("/")); - return ObjectStorageKey::createAsRelative(fs::path(url_without_path) / data_directory.substr(1) / path); + return ObjectStorageKey::createAsRelative(fs::path(url_without_path) / data_directory.substr(1), path); } } #endif diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h index a532c98cb87d..da37e1d782db 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h @@ -33,7 +33,7 @@ class GlutenHDFSObjectStorage final : public DB::HDFSObjectStorage const String & hdfs_root_path_, SettingsPtr settings_, const Poco::Util::AbstractConfiguration & config_) - : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_, /* lazy_initialize */true), config(config_) + : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_, /* lazy_initialize */false) { } std::unique_ptr readObject( /// NOLINT @@ -43,8 +43,6 @@ class GlutenHDFSObjectStorage final : public DB::HDFSObjectStorage std::optional file_size = {}) const override; DB::ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override; hdfsFS getHDFSFS() const { return hdfs_fs.get(); } -private: - const Poco::Util::AbstractConfiguration & config; }; #endif diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp index c080e0525f3c..9e4546498034 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp @@ -121,7 +121,7 @@ void registerGlutenHDFSObjectStorage(ObjectStorageFactory & factory) config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), context->getSettingsRef().hdfs_replication ); - return std::make_unique(uri, std::move(settings), config); + return std::make_shared(uri, std::move(settings), config); }); } #endif diff --git a/cpp-ch/local-engine/Disks/registerGlutenDisks.cpp b/cpp-ch/local-engine/Disks/registerGlutenDisks.cpp index c7e9c5fd32ba..8a920edcce77 100644 --- a/cpp-ch/local-engine/Disks/registerGlutenDisks.cpp +++ b/cpp-ch/local-engine/Disks/registerGlutenDisks.cpp @@ -40,6 +40,8 @@ void registerGlutenHDFSObjectStorage(DB::ObjectStorageFactory & factory); void registerGlutenDisks(bool global_skip_access_check) { auto & factory = DB::DiskFactory::instance(); + +#if USE_AWS_S3 auto creator = [global_skip_access_check]( const String & name, const Poco::Util::AbstractConfiguration & config, @@ -66,7 +68,7 @@ void registerGlutenDisks(bool global_skip_access_check) }; auto & object_factory = DB::ObjectStorageFactory::instance(); -#if USE_AWS_S3 + registerGlutenS3ObjectStorage(object_factory); factory.registerDiskType("s3_gluten", creator); /// For compatibility #endif @@ -82,11 +84,20 @@ void registerGlutenDisks(bool global_skip_access_check) bool) -> DB::DiskPtr { bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false); - auto object_storage = DB::ObjectStorageFactory::instance().create(name, config, config_prefix, context, skip_access_check); + auto object_storage_creator = [name, skip_access_check, config_prefix]( + const Poco::Util::AbstractConfiguration & conf, DB::ContextPtr ctx) -> DB::ObjectStoragePtr + { return DB::ObjectStorageFactory::instance().create(name, conf, config_prefix, ctx, skip_access_check); }; + auto object_storage = object_storage_creator(config, context); auto metadata_storage = DB::MetadataStorageFactory::instance().create(name, config, config_prefix, object_storage, "local"); DB::DiskObjectStoragePtr disk = std::make_shared( - name, object_storage->getCommonKeyPrefix(), std::move(metadata_storage), std::move(object_storage), config, config_prefix); + name, + object_storage->getCommonKeyPrefix(), + std::move(metadata_storage), + std::move(object_storage), + config, + config_prefix, + object_storage_creator); disk->startup(context, skip_access_check); return disk; From 7ac9983cde6ed2f942eaf05c628055bc715b6990 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 19 Jun 2024 13:07:22 +0800 Subject: [PATCH 294/402] [VL] Gluten-it: Reuse Spark sessions that share same configuration (#6117) --- .github/workflows/velox_docker.yml | 2 +- .../integration/command/Parameterized.java | 19 +- .../gluten/integration/command/Queries.java | 2 +- .../integration/command/QueriesCompare.java | 2 +- .../integration/command/QueriesMixin.java | 7 + .../gluten/integration/QueryRunner.scala | 53 ++- .../integration/action/Parameterized.scala | 350 +++++++++--------- .../gluten/integration/action/Queries.scala | 148 +++----- .../integration/action/QueriesCompare.scala | 222 ++++++----- .../integration/action/SparkShell.scala | 2 +- .../gluten/integration/action/package.scala | 28 +- .../clickbench/ClickBenchDataGen.scala | 5 +- .../spark/sql/SparkSessionSwitcher.scala | 10 + 13 files changed, 440 insertions(+), 410 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 6c1be4344c71..b1d5cfdcf3d3 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -367,7 +367,7 @@ jobs: cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries \ --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ - --skip-data-gen --random-kill-tasks + --skip-data-gen --random-kill-tasks --no-session-reuse # run-tpc-test-ubuntu-sf30: # needs: build-native-lib-centos-7 diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java index cadff0a2db91..225b492ef1ae 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Parameterized.java @@ -18,9 +18,6 @@ import com.google.common.base.Preconditions; import org.apache.gluten.integration.BaseMixin; -import org.apache.gluten.integration.action.Dim; -import org.apache.gluten.integration.action.DimKv; -import org.apache.gluten.integration.action.DimValue; import org.apache.commons.lang3.ArrayUtils; import picocli.CommandLine; import scala.Tuple2; @@ -67,17 +64,17 @@ public class Parameterized implements Callable { public Integer call() throws Exception { final Map>>> parsed = new LinkedHashMap<>(); - final Seq> excludedCombinations = JavaConverters.asScalaBufferConverter(Arrays.stream(excludedDims).map(d -> { + final Seq> excludedCombinations = JavaConverters.asScalaBufferConverter(Arrays.stream(excludedDims).map(d -> { final Matcher m = excludedDimsPattern.matcher(d); Preconditions.checkArgument(m.matches(), "Unrecognizable excluded dims: " + d); - Set out = new HashSet<>(); + Set out = new HashSet<>(); final String[] dims = d.split(","); for (String dim : dims) { final String[] kv = dim.split(":"); Preconditions.checkArgument(kv.length == 2, "Unrecognizable excluded dims: " + d); - out.add(new DimKv(kv[0], kv[1])); + out.add(new org.apache.gluten.integration.action.Parameterized.DimKv(kv[0], kv[1])); } - return JavaConverters.asScalaSetConverter(out).asScala().toSet(); + return JavaConverters.asScalaSetConverter(out).asScala().toSet(); }).collect(Collectors.toList())).asScala(); // parse dims @@ -121,11 +118,11 @@ public Integer call() throws Exception { } // Convert Map>>> to List - Seq parsedDims = JavaConverters.asScalaBufferConverter( + Seq parsedDims = JavaConverters.asScalaBufferConverter( parsed.entrySet().stream().map(e -> - new Dim(e.getKey(), JavaConverters.asScalaBufferConverter( + new org.apache.gluten.integration.action.Parameterized.Dim(e.getKey(), JavaConverters.asScalaBufferConverter( e.getValue().entrySet().stream().map(e2 -> - new DimValue(e2.getKey(), JavaConverters.asScalaBufferConverter( + new org.apache.gluten.integration.action.Parameterized.DimValue(e2.getKey(), JavaConverters.asScalaBufferConverter( e2.getValue().stream().map(e3 -> new Tuple2<>(e3.getKey(), e3.getValue())) .collect(Collectors.toList())).asScala())).collect(Collectors.toList())).asScala() )).collect(Collectors.toList())).asScala(); @@ -133,7 +130,7 @@ public Integer call() throws Exception { org.apache.gluten.integration.action.Parameterized parameterized = new org.apache.gluten.integration.action.Parameterized(dataGenMixin.getScale(), dataGenMixin.genPartitionedData(), queriesMixin.queries(), - queriesMixin.explain(), queriesMixin.iterations(), warmupIterations, parsedDims, + queriesMixin.explain(), queriesMixin.iterations(), warmupIterations, queriesMixin.noSessionReuse(), parsedDims, excludedCombinations, metrics); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), parameterized)); } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java index f0c07b41538b..c19d66bdae75 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/Queries.java @@ -42,7 +42,7 @@ public class Queries implements Callable { public Integer call() throws Exception { org.apache.gluten.integration.action.Queries queries = new org.apache.gluten.integration.action.Queries(dataGenMixin.getScale(), dataGenMixin.genPartitionedData(), queriesMixin.queries(), - queriesMixin.explain(), queriesMixin.iterations(), randomKillTasks); + queriesMixin.explain(), queriesMixin.iterations(), randomKillTasks, queriesMixin.noSessionReuse()); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), queries)); } } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java index 42b00f94cece..d194aad185a9 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesCompare.java @@ -40,7 +40,7 @@ public Integer call() throws Exception { org.apache.gluten.integration.action.QueriesCompare queriesCompare = new org.apache.gluten.integration.action.QueriesCompare(dataGenMixin.getScale(), dataGenMixin.genPartitionedData(), queriesMixin.queries(), - queriesMixin.explain(), queriesMixin.iterations()); + queriesMixin.explain(), queriesMixin.iterations(), queriesMixin.noSessionReuse()); return mixin.runActions(ArrayUtils.addAll(dataGenMixin.makeActions(), queriesCompare)); } } diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java index fc93f968c85c..64e4b32eca5e 100644 --- a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java +++ b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/command/QueriesMixin.java @@ -42,6 +42,9 @@ public class QueriesMixin { @CommandLine.Option(names = {"--iterations"}, description = "How many iterations to run", defaultValue = "1") private int iterations; + @CommandLine.Option(names = {"--no-session-reuse"}, description = "Recreate new Spark session each time a query is about to run", defaultValue = "false") + private boolean noSessionReuse; + public boolean explain() { return explain; } @@ -50,6 +53,10 @@ public int iterations() { return iterations; } + public boolean noSessionReuse() { + return noSessionReuse; + } + public Actions.QuerySelector queries() { return new Actions.QuerySelector() { @Override diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala index 88e8e2250fd8..9791242f18cd 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/QueryRunner.scala @@ -17,11 +17,14 @@ package org.apache.gluten.integration import com.google.common.base.Preconditions +import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.spark.sql.{RunResult, SparkQueryRunner, SparkSession} import java.io.File class QueryRunner(val queryResourceFolder: String, val dataPath: String) { + import QueryRunner._ + Preconditions.checkState( new File(dataPath).exists(), s"Data not found at $dataPath, try using command ` data-gen-only ` to generate it first.", @@ -37,10 +40,54 @@ class QueryRunner(val queryResourceFolder: String, val dataPath: String) { caseId: String, explain: Boolean = false, metrics: Array[String] = Array(), - randomKillTasks: Boolean = false): RunResult = { + randomKillTasks: Boolean = false): QueryResult = { val path = "%s/%s.sql".format(queryResourceFolder, caseId) - SparkQueryRunner.runQuery(spark, desc, path, explain, metrics, randomKillTasks) + try { + val r = SparkQueryRunner.runQuery(spark, desc, path, explain, metrics, randomKillTasks) + println(s"Successfully ran query $caseId. Returned row count: ${r.rows.length}") + Success(caseId, r) + } catch { + case e: Exception => + println(s"Error running query $caseId. Error: ${ExceptionUtils.getStackTrace(e)}") + Failure(caseId, e) + } } } -object QueryRunner {} +object QueryRunner { + sealed trait QueryResult { + def caseId(): String + def succeeded(): Boolean + } + + implicit class QueryResultOps(r: QueryResult) { + def asSuccessOption(): Option[Success] = { + r match { + case s: Success => Some(s) + case _: Failure => None + } + } + + def asFailureOption(): Option[Failure] = { + r match { + case _: Success => None + case f: Failure => Some(f) + } + } + + def asSuccess(): Success = { + asSuccessOption().get + } + + def asFailure(): Failure = { + asFailureOption().get + } + } + + case class Success(override val caseId: String, runResult: RunResult) extends QueryResult { + override def succeeded(): Boolean = true + } + case class Failure(override val caseId: String, error: Exception) extends QueryResult { + override def succeeded(): Boolean = false + } +} diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala index 74f22a05f5fe..c9ebb9754394 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Parameterized.scala @@ -17,15 +17,16 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.gluten.integration.QueryRunner.QueryResult import org.apache.gluten.integration.action.Actions.QuerySelector import org.apache.gluten.integration.action.TableRender.Field import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat -import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} +import org.apache.gluten.integration.{QueryRunner, Suite} import org.apache.spark.sql.ConfUtils.ConfImplicits._ -import org.apache.spark.sql.SparkSessionSwitcher +import org.apache.spark.sql.SparkSession -import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.{AtomicInteger, AtomicLong} import scala.collection.mutable import scala.collection.mutable.ListBuffer @@ -36,20 +37,22 @@ class Parameterized( explain: Boolean, iterations: Int, warmupIterations: Int, - configDimensions: Seq[Dim], - excludedCombinations: Seq[Set[DimKv]], + noSessionReuse: Boolean, + configDimensions: Seq[Parameterized.Dim], + excludedCombinations: Seq[Set[Parameterized.DimKv]], metrics: Array[String]) extends Action { + import Parameterized._ validateDims(configDimensions) private def validateDims(configDimensions: Seq[Dim]): Unit = { if (configDimensions - .map(dim => { - dim.name - }) - .toSet - .size != configDimensions.size) { + .map(dim => { + dim.name + }) + .toSet + .size != configDimensions.size) { throw new IllegalStateException("Duplicated dimension name found") } @@ -73,9 +76,9 @@ class Parameterized( // we got one coordinate excludedCombinations.foreach { ec: Set[DimKv] => if (ec.forall { kv => - intermediateCoordinate.contains(kv.k) && intermediateCoordinate(kv.k) == kv.v - }) { - println(s"Coordinate ${intermediateCoordinate} excluded by $ec.") + intermediateCoordinate.contains(kv.k) && intermediateCoordinate(kv.k) == kv.v + }) { + println(s"Coordinate $intermediateCoordinate excluded by $ec.") return } } @@ -105,9 +108,8 @@ class Parameterized( val testConf = suite.getTestConf() println("Prepared coordinates: ") - coordinates.toList.map(_._1).zipWithIndex.foreach { - case (c, idx) => - println(s" $idx: $c") + coordinates.keys.foreach { c => + println(s" ${c.id}: $c") } coordinates.foreach { entry => // register one session per coordinate @@ -118,39 +120,67 @@ class Parameterized( sessionSwitcher.registerSession(coordinate.toString, conf) } - val runQueryIds = queries.select(suite) + val runQueryIds = queries.select(suite).map(TestResultLine.QueryId(_)) - val results = (0 until iterations).flatMap { iteration => - runQueryIds.map { queryId => - val queryResult = - TestResultLine( - queryId, - coordinates.map { entry => - val coordinate = entry._1 - println(s"Running tests (iteration $iteration) with coordinate $coordinate...") - // warm up - (0 until warmupIterations).foreach { _ => - Parameterized.warmUp( - runner, - suite.tableCreator(), - sessionSwitcher, - queryId, - suite.desc()) - } - // run + val marks: Seq[TestResultLine.CoordMark] = coordinates.flatMap { entry => + val coordinate = entry._1 + sessionSwitcher.useSession(coordinate.toString, "Parameterized %s".format(coordinate)) + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + + runQueryIds.flatMap { queryId => + // warm up + (0 until warmupIterations).foreach { iteration => + println(s"Warming up: Running query $queryId (iteration $iteration)...") + try { + Parameterized.warmUp( + runner, + sessionSwitcher.spark(), + queryId.id, + coordinate, + suite.desc()) + } finally { + if (noSessionReuse) { + sessionSwitcher.renewSession() + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + } + } + } + + // run + (0 until iterations).map { iteration => + println(s"Running query $queryId with coordinate $coordinate (iteration $iteration)...") + val r = + try { Parameterized.runQuery( runner, - suite.tableCreator(), - sessionSwitcher, - queryId, + sessionSwitcher.spark(), + queryId.id, coordinate, suite.desc(), explain, metrics) - }.toList) - queryResult + } finally { + if (noSessionReuse) { + sessionSwitcher.renewSession() + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + } + } + TestResultLine.CoordMark(iteration, queryId, r) + } + } + }.toSeq + + val results: Seq[TestResultLine] = marks + .groupBy(m => (m.iteration, m.queryId)) + .toSeq + .sortBy(_._1) + .map { e => + val iteration = e._1._1 + val queryId = e._1._2 + val marks = e._2 + val line = TestResultLine(queryId, marks.map(_.coord).toList) + line } - } val succeededCount = results.count(l => l.succeeded()) val totalCount = results.count(_ => true) @@ -174,16 +204,27 @@ class Parameterized( totalCount) println("") println("Configurations:") - coordinates.foreach { coord => - println(s"${coord._1.id}. ${coord._1}") - } + coordinates.foreach(coord => println(s"${coord._1.id}. ${coord._1}")) println("") val succeeded = results.filter(_.succeeded()) - TestResultLines( - coordinates.size, - configDimensions, - metrics, - succeeded ++ TestResultLine.aggregate("all", succeeded)) + val all = succeeded match { + case Nil => None + case several => + Some( + TestResultLine( + TestResultLine.QueryId("all"), + coordinates.keys.map { c => + TestResultLine.Coord( + c, + several + .map(_.coord(c.id)) + .map(_.queryResult) + .asSuccesses() + .agg(s"coordinate $c") + .get) + }.toSeq)) + } + TestResultLines(coordinates.map(_._1.id).toSeq, configDimensions, metrics, succeeded ++ all) .print() println("") @@ -193,7 +234,11 @@ class Parameterized( } else { println("Failed queries: ") println("") - TestResultLines(coordinates.size, configDimensions, metrics, results.filter(!_.succeeded())) + TestResultLines( + coordinates.map(_._1.id).toSeq, + configDimensions, + metrics, + results.filter(!_.succeeded())) .print() println("") } @@ -205,157 +250,114 @@ class Parameterized( } } -case class DimKv(k: String, v: String) -case class Dim(name: String, dimValues: Seq[DimValue]) -case class DimValue(name: String, conf: Seq[(String, String)]) -// coordinate: [dim, dim value] -case class Coordinate(id: Int, coordinate: Map[String, String]) { - override def toString: String = coordinate.mkString(", ") -} +object Parameterized { + case class DimKv(k: String, v: String) + + case class Dim(name: String, dimValues: Seq[DimValue]) + + case class DimValue(name: String, conf: Seq[(String, String)]) -case class TestResultLine(queryId: String, coordinates: Seq[TestResultLine.Coord]) { - def succeeded(): Boolean = { - coordinates.forall(_.succeeded) + // coordinate: [dim, dim value] + case class Coordinate(id: Int, coordinate: Map[String, String]) { + override def toString: String = coordinate.mkString(", ") } -} -object TestResultLine { - case class Coord( - coordinate: Coordinate, - succeeded: Boolean, - rowCount: Option[Long], - planningTimeMillis: Option[Long], - executionTimeMillis: Option[Long], - metrics: Map[String, Long], - errorMessage: Option[String]) - - class Parser(metricNames: Seq[String]) extends TableRender.RowParser[TestResultLine] { - override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { - val inc = rowAppender.incremental() - inc.next().write(line.queryId) - val coords = line.coordinates - coords.foreach(coord => inc.next().write(coord.succeeded)) - coords.foreach(coord => inc.next().write(coord.rowCount)) - metricNames.foreach(metricName => - coords.foreach(coord => inc.next().write(coord.metrics(metricName)))) - coords.foreach(coord => inc.next().write(coord.planningTimeMillis)) - coords.foreach(coord => inc.next().write(coord.executionTimeMillis)) + case class TestResultLine( + queryId: TestResultLine.QueryId, + coordinates: Seq[TestResultLine.Coord]) { + private val coordMap = coordinates.map(c => c.coordinate.id -> c).toMap + def succeeded(): Boolean = { + coordinates.forall(_.queryResult.succeeded()) } + + def coord(id: Int): TestResultLine.Coord = coordMap(id) } - def aggregate(name: String, lines: Iterable[TestResultLine]): Iterable[TestResultLine] = { - if (lines.isEmpty) { - return Nil + object TestResultLine { + case class QueryId(id: String) { + import QueryId._ + private val uid = nextUid.getAndIncrement() + override def toString: String = id } - if (lines.size == 1) { - return Nil + object QueryId { + private val nextUid = new AtomicLong(0L) + implicit val o: Ordering[QueryId] = Ordering.by(_.uid) } - List(lines.reduce { (left, right) => - TestResultLine(name, left.coordinates.zip(right.coordinates).map { - case (leftCoord, rightCoord) => - assert(leftCoord.coordinate == rightCoord.coordinate) - Coord( - leftCoord.coordinate, - leftCoord.succeeded && rightCoord.succeeded, - (leftCoord.rowCount, rightCoord.rowCount).onBothProvided(_ + _), - (leftCoord.planningTimeMillis, rightCoord.planningTimeMillis).onBothProvided(_ + _), - (leftCoord.executionTimeMillis, rightCoord.executionTimeMillis).onBothProvided(_ + _), - (leftCoord.metrics, rightCoord.metrics).sumUp, - (leftCoord.errorMessage ++ rightCoord.errorMessage).reduceOption(_ + ", " + _)) - }) - }) + case class Coord(coordinate: Coordinate, queryResult: QueryResult) + case class CoordMark(iteration: Int, queryId: QueryId, coord: Coord) + + class Parser(coordIds: Seq[Int], metricNames: Seq[String]) + extends TableRender.RowParser[TestResultLine] { + override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { + val inc = rowAppender.incremental() + inc.next().write(line.queryId) + val coords = coordIds.map(id => line.coord(id)) + coords.foreach(coord => inc.next().write(coord.queryResult.succeeded())) + coords.foreach(coord => + inc.next().write(coord.queryResult.asSuccessOption().map(_.runResult.rows.size))) + metricNames.foreach(metricName => + coords.foreach(coord => + inc + .next() + .write(coord.queryResult.asSuccessOption().map(_.runResult.metrics(metricName))))) + coords.foreach(coord => + inc + .next() + .write(coord.queryResult.asSuccessOption().map(_.runResult.planningTimeMillis))) + coords.foreach(coord => + inc + .next() + .write(coord.queryResult.asSuccessOption().map(_.runResult.executionTimeMillis))) + } + } } -} -case class TestResultLines( - coordCount: Int, - configDimensions: Seq[Dim], - metricNames: Seq[String], - lines: Iterable[TestResultLine]) { - def print(): Unit = { - val fields = ListBuffer[Field](Field.Leaf("Query ID")) - val coordFields = (1 to coordCount).map(id => Field.Leaf(id.toString)) - - fields.append(Field.Branch("Succeeded", coordFields)) - fields.append(Field.Branch("Row Count", coordFields)) - metricNames.foreach(metricName => fields.append(Field.Branch(metricName, coordFields))) - fields.append(Field.Branch("Planning Time (Millis)", coordFields)) - fields.append(Field.Branch("Query Time (Millis)", coordFields)) - - val render = - TableRender.create[TestResultLine](fields: _*)(new TestResultLine.Parser(metricNames)) - - lines.foreach { line => - render.appendRow(line) - } + case class TestResultLines( + coordIds: Seq[Int], + configDimensions: Seq[Dim], + metricNames: Seq[String], + lines: Iterable[TestResultLine]) { + def print(): Unit = { + val fields = ListBuffer[Field](Field.Leaf("Query ID")) + val coordFields = coordIds.map(id => Field.Leaf(id.toString)) + + fields.append(Field.Branch("Succeeded", coordFields)) + fields.append(Field.Branch("Row Count", coordFields)) + metricNames.foreach(metricName => fields.append(Field.Branch(metricName, coordFields))) + fields.append(Field.Branch("Planning Time (Millis)", coordFields)) + fields.append(Field.Branch("Query Time (Millis)", coordFields)) - render.print(System.out) + val render = + TableRender.create[TestResultLine](fields: _*)( + new TestResultLine.Parser(coordIds, metricNames)) + + lines.foreach(line => render.appendRow(line)) + + render.print(System.out) + } } -} -object Parameterized { private def runQuery( runner: QueryRunner, - creator: TableCreator, - sessionSwitcher: SparkSessionSwitcher, + spark: SparkSession, id: String, coordinate: Coordinate, desc: String, explain: Boolean, metrics: Array[String]): TestResultLine.Coord = { - println(s"Running query: $id...") - try { - val testDesc = "Gluten Spark %s [%s] %s".format(desc, id, coordinate) - sessionSwitcher.useSession(coordinate.toString, testDesc) - runner.createTables(creator, sessionSwitcher.spark()) - val result = - runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain, metrics) - val resultRows = result.rows - println( - s"Successfully ran query $id. " + - s"Returned row count: ${resultRows.length}") - TestResultLine.Coord( - coordinate, - succeeded = true, - Some(resultRows.length), - Some(result.planningTimeMillis), - Some(result.executionTimeMillis), - result.metrics, - None) - } catch { - case e: Exception => - val error = Some(s"FATAL: ${ExceptionUtils.getStackTrace(e)}") - println( - s"Error running query $id. " + - s" Error: ${error.get}") - TestResultLine.Coord(coordinate, succeeded = false, None, None, None, Map.empty, error) - } + val testDesc = "Query %s [%s] %s".format(desc, id, coordinate) + val result = runner.runQuery(spark, testDesc, id, explain, metrics) + TestResultLine.Coord(coordinate, result) } private def warmUp( runner: QueryRunner, - creator: TableCreator, - sessionSwitcher: SparkSessionSwitcher, + session: SparkSession, id: String, + coordinate: Coordinate, desc: String): Unit = { - println(s"Warming up: Running query: $id...") - try { - val testDesc = "Gluten Spark %s [%s] Warm Up".format(desc, id) - sessionSwitcher.useSession("test", testDesc) - runner.createTables(creator, sessionSwitcher.spark()) - val result = runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain = false) - val resultRows = result.rows - println( - s"Warming up: Successfully ran query $id. " + - s"Returned row count: ${resultRows.length}") - } catch { - case e: Exception => - val error = Some(s"FATAL: ${ExceptionUtils.getStackTrace(e)}") - println( - s"Warming up: Error running query $id. " + - s" Error: ${error.get}") - } + runQuery(runner, session, id, coordinate, desc, explain = false, Array.empty) } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala index de09d925e4d2..b8a42f393932 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/Queries.scala @@ -16,11 +16,12 @@ */ package org.apache.gluten.integration.action -import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.gluten.integration.QueryRunner.QueryResult import org.apache.gluten.integration.action.Actions.QuerySelector import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat -import org.apache.gluten.integration.{QueryRunner, Suite} +import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} +import org.apache.spark.sql.{SparkSession} case class Queries( scale: Double, @@ -28,28 +29,40 @@ case class Queries( queries: QuerySelector, explain: Boolean, iterations: Int, - randomKillTasks: Boolean) + randomKillTasks: Boolean, + noSessionReuse: Boolean) extends Action { + import Queries._ override def execute(suite: Suite): Boolean = { val runQueryIds = queries.select(suite) val runner: QueryRunner = new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) + val sessionSwitcher = suite.sessionSwitcher + sessionSwitcher.useSession("test", "Run Queries") + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) val results = (0 until iterations).flatMap { iteration => println(s"Running tests (iteration $iteration)...") runQueryIds.map { queryId => - Queries.runQuery( - runner, - suite.tableCreator(), - suite.sessionSwitcher, - queryId, - suite.desc(), - explain, - randomKillTasks) + try { + Queries.runQuery( + runner, + suite.tableCreator(), + sessionSwitcher.spark(), + queryId, + suite.desc(), + explain, + randomKillTasks) + } finally { + if (noSessionReuse) { + sessionSwitcher.renewSession() + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + } + } } }.toList - val passedCount = results.count(l => l.testPassed) + val passedCount = results.count(l => l.queryResult.succeeded()) val count = results.count(_ => true) // RAM stats @@ -67,8 +80,9 @@ case class Queries( println("") printf("Summary: %d out of %d queries passed. \n", passedCount, count) println("") - val succeed = results.filter(_.testPassed) - Queries.printResults(succeed) + val succeeded = results.filter(_.queryResult.succeeded()) + val all = succeeded.map(_.queryResult).asSuccesses().agg("all").map(s => TestResultLine(s)) + Queries.printResults(succeeded ++ all) println("") if (passedCount == count) { @@ -77,21 +91,10 @@ case class Queries( } else { println("Failed queries: ") println("") - Queries.printResults(results.filter(!_.testPassed)) + Queries.printResults(results.filter(!_.queryResult.succeeded())) println("") } - var all = Queries.aggregate(results, "all") - - if (passedCount != count) { - all = Queries.aggregate(succeed, "succeeded") ::: all - } - - println("Overall: ") - println("") - Queries.printResults(all) - println("") - if (passedCount != count) { return false } @@ -100,28 +103,29 @@ case class Queries( } object Queries { - case class TestResultLine( - queryId: String, - testPassed: Boolean, - rowCount: Option[Long], - planningTimeMillis: Option[Long], - executionTimeMillis: Option[Long], - errorMessage: Option[String]) + case class TestResultLine(queryResult: QueryResult) object TestResultLine { implicit object Parser extends TableRender.RowParser[TestResultLine] { override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { val inc = rowAppender.incremental() - inc.next().write(line.queryId) - inc.next().write(line.testPassed) - inc.next().write(line.rowCount) - inc.next().write(line.planningTimeMillis) - inc.next().write(line.executionTimeMillis) + inc.next().write(line.queryResult.caseId()) + inc.next().write(line.queryResult.succeeded()) + line.queryResult match { + case QueryRunner.Success(_, runResult) => + inc.next().write(runResult.rows.size) + inc.next().write(runResult.planningTimeMillis) + inc.next().write(runResult.executionTimeMillis) + case QueryRunner.Failure(_, error) => + inc.next().write(None) + inc.next().write(None) + inc.next().write(None) + } } } } - private def printResults(results: List[TestResultLine]): Unit = { + private def printResults(results: Seq[TestResultLine]): Unit = { val render = TableRender.plain[TestResultLine]( "Query ID", "Was Passed", @@ -136,64 +140,18 @@ object Queries { render.print(System.out) } - private def aggregate(succeed: List[TestResultLine], name: String): List[TestResultLine] = { - if (succeed.isEmpty) { - return Nil - } - List( - succeed.reduce((r1, r2) => - TestResultLine( - name, - testPassed = true, - if (r1.rowCount.nonEmpty && r2.rowCount.nonEmpty) - Some(r1.rowCount.get + r2.rowCount.get) - else None, - if (r1.planningTimeMillis.nonEmpty && r2.planningTimeMillis.nonEmpty) - Some(r1.planningTimeMillis.get + r2.planningTimeMillis.get) - else None, - if (r1.executionTimeMillis.nonEmpty && r2.executionTimeMillis.nonEmpty) - Some(r1.executionTimeMillis.get + r2.executionTimeMillis.get) - else None, - None))) - } - private def runQuery( - runner: _root_.org.apache.gluten.integration.QueryRunner, - creator: _root_.org.apache.gluten.integration.TableCreator, - sessionSwitcher: _root_.org.apache.spark.sql.SparkSessionSwitcher, - id: _root_.java.lang.String, - desc: _root_.java.lang.String, + runner: QueryRunner, + creator: TableCreator, + session: SparkSession, + id: String, + desc: String, explain: Boolean, - randomKillTasks: Boolean) = { + randomKillTasks: Boolean): TestResultLine = { println(s"Running query: $id...") - try { - val testDesc = "Gluten Spark %s %s".format(desc, id) - sessionSwitcher.useSession("test", testDesc) - runner.createTables(creator, sessionSwitcher.spark()) - val result = runner.runQuery( - sessionSwitcher.spark(), - testDesc, - id, - explain = explain, - randomKillTasks = randomKillTasks) - val resultRows = result.rows - println( - s"Successfully ran query $id. " + - s"Returned row count: ${resultRows.length}") - TestResultLine( - id, - testPassed = true, - Some(resultRows.length), - Some(result.planningTimeMillis), - Some(result.executionTimeMillis), - None) - } catch { - case e: Exception => - val error = Some(s"FATAL: ${ExceptionUtils.getStackTrace(e)}") - println( - s"Error running query $id. " + - s" Error: ${error.get}") - TestResultLine(id, testPassed = false, None, None, None, error) - } + val testDesc = "Query %s [%s]".format(desc, id) + val result = + runner.runQuery(session, testDesc, id, explain = explain, randomKillTasks = randomKillTasks) + TestResultLine(result) } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala index d7b6ffff893c..804f1fbd79f3 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/QueriesCompare.scala @@ -17,37 +17,78 @@ package org.apache.gluten.integration.action import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.gluten.integration.QueryRunner.QueryResult import org.apache.gluten.integration.action.Actions.QuerySelector +import org.apache.gluten.integration.action.QueriesCompare.TestResultLine import org.apache.gluten.integration.action.TableRender.RowParser.FieldAppender.RowAppender import org.apache.gluten.integration.stat.RamStat import org.apache.gluten.integration.{QueryRunner, Suite, TableCreator} -import org.apache.spark.sql.{SparkSessionSwitcher, TestUtils} +import org.apache.spark.sql.{RunResult, SparkSession, SparkSessionSwitcher, TestUtils} case class QueriesCompare( scale: Double, genPartitionedData: Boolean, queries: QuerySelector, explain: Boolean, - iterations: Int) + iterations: Int, + noSessionReuse: Boolean) extends Action { override def execute(suite: Suite): Boolean = { val runner: QueryRunner = new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) val runQueryIds = queries.select(suite) - val results = (0 until iterations).flatMap { iteration => - println(s"Running tests (iteration $iteration)...") + val sessionSwitcher = suite.sessionSwitcher + + sessionSwitcher.useSession("baseline", "Run Baseline Queries") + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + val baselineResults = (0 until iterations).flatMap { iteration => + runQueryIds.map { queryId => + println(s"Running baseline query $queryId (iteration $iteration)...") + try { + QueriesCompare.runBaselineQuery( + runner, + sessionSwitcher.spark(), + suite.desc(), + queryId, + explain) + } finally { + if (noSessionReuse) { + sessionSwitcher.renewSession() + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + } + } + } + }.toList + + sessionSwitcher.useSession("test", "Run Test Queries") + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + val testResults = (0 until iterations).flatMap { iteration => runQueryIds.map { queryId => - QueriesCompare.runQuery( - suite.tableCreator(), - queryId, - explain, - suite.desc(), - suite.sessionSwitcher, - runner) + println(s"Running test query $queryId (iteration $iteration)...") + try { + QueriesCompare.runTestQuery( + runner, + sessionSwitcher.spark(), + suite.desc(), + queryId, + explain) + } finally { + if (noSessionReuse) { + sessionSwitcher.renewSession() + runner.createTables(suite.tableCreator(), sessionSwitcher.spark()) + } + } } }.toList + assert(baselineResults.size == testResults.size) + + val results: Seq[TestResultLine] = baselineResults.zip(testResults).map { case (b, t) => + assert(b.caseId() == t.caseId()) + TestResultLine(b.caseId(), b, t) + } + val passedCount = results.count(l => l.testPassed) val count = results.count(_ => true) @@ -66,8 +107,15 @@ case class QueriesCompare( println("") printf("Summary: %d out of %d queries passed. \n", passedCount, count) println("") - val succeed = results.filter(_.testPassed) - QueriesCompare.printResults(succeed) + val succeeded = results.filter(_.testPassed) + val all = succeeded match { + case Nil => None + case several => + val allExpected = several.map(_.expected).asSuccesses().agg("all expected").get + val allActual = several.map(_.actual).asSuccesses().agg("all actual").get + Some(TestResultLine("all", allExpected, allActual)) + } + QueriesCompare.printResults(succeeded ++ all) println("") if (passedCount == count) { @@ -81,17 +129,6 @@ case class QueriesCompare( println("") } - var all = QueriesCompare.aggregate("all", results) - - if (passedCount != count) { - all = QueriesCompare.aggregate("succeeded", succeed) ::: all - } - - println("Overall: ") - println("") - QueriesCompare.printResults(all) - println("") - if (passedCount != count) { return false } @@ -100,41 +137,46 @@ case class QueriesCompare( } object QueriesCompare { - case class TestResultLine( - queryId: String, - testPassed: Boolean, - expectedRowCount: Option[Long], - actualRowCount: Option[Long], - expectedPlanningTimeMillis: Option[Long], - actualPlanningTimeMillis: Option[Long], - expectedExecutionTimeMillis: Option[Long], - actualExecutionTimeMillis: Option[Long], - errorMessage: Option[String]) + case class TestResultLine(queryId: String, expected: QueryResult, actual: QueryResult) { + val testPassed: Boolean = { + expected.succeeded() && actual.succeeded() && + TestUtils + .compareAnswers( + expected.asSuccess().runResult.rows, + actual.asSuccess().runResult.rows, + sort = true) + .isEmpty + } + } object TestResultLine { implicit object Parser extends TableRender.RowParser[TestResultLine] { override def parse(rowAppender: RowAppender, line: TestResultLine): Unit = { val inc = rowAppender.incremental() + inc.next().write(line.queryId) + inc.next().write(line.testPassed) + inc.next().write(line.expected.asSuccessOption().map(_.runResult.rows.size)) + inc.next().write(line.actual.asSuccessOption().map(_.runResult.rows.size)) + inc.next().write(line.expected.asSuccessOption().map(_.runResult.planningTimeMillis)) + inc.next().write(line.actual.asSuccessOption().map(_.runResult.planningTimeMillis)) + inc.next().write(line.expected.asSuccessOption().map(_.runResult.executionTimeMillis)) + inc.next().write(line.actual.asSuccessOption().map(_.runResult.executionTimeMillis)) + val speedUp = - if (line.expectedExecutionTimeMillis.nonEmpty && line.actualExecutionTimeMillis.nonEmpty) { + if (line.expected.succeeded() && line.actual.succeeded()) { Some( - ((line.expectedExecutionTimeMillis.get - line.actualExecutionTimeMillis.get).toDouble - / line.actualExecutionTimeMillis.get.toDouble) * 100) + ((line.expected.asSuccess().runResult.executionTimeMillis - line.actual + .asSuccess() + .runResult + .executionTimeMillis).toDouble + / line.actual.asSuccess().runResult.executionTimeMillis) * 100) } else None - inc.next().write(line.queryId) - inc.next().write(line.testPassed) - inc.next().write(line.expectedRowCount) - inc.next().write(line.actualRowCount) - inc.next().write(line.expectedPlanningTimeMillis) - inc.next().write(line.actualPlanningTimeMillis) - inc.next().write(line.expectedExecutionTimeMillis) - inc.next().write(line.actualExecutionTimeMillis) inc.next().write(speedUp.map("%.2f%%".format(_))) } } } - private def printResults(results: List[TestResultLine]): Unit = { + private def printResults(results: Seq[TestResultLine]): Unit = { import org.apache.gluten.integration.action.TableRender.Field._ val render = TableRender.create[TestResultLine]( @@ -152,79 +194,25 @@ object QueriesCompare { render.print(System.out) } - private def aggregate(name: String, succeed: List[TestResultLine]): List[TestResultLine] = { - if (succeed.isEmpty) { - return Nil - } - List( - succeed.reduce((r1, r2) => - TestResultLine( - name, - r1.testPassed && r2.testPassed, - (r1.expectedRowCount, r2.expectedRowCount).onBothProvided(_ + _), - (r1.actualRowCount, r2.actualRowCount).onBothProvided(_ + _), - (r1.expectedPlanningTimeMillis, r2.expectedPlanningTimeMillis).onBothProvided(_ + _), - (r1.actualPlanningTimeMillis, r2.actualPlanningTimeMillis).onBothProvided(_ + _), - (r1.expectedExecutionTimeMillis, r2.expectedExecutionTimeMillis).onBothProvided(_ + _), - (r1.actualExecutionTimeMillis, r2.actualExecutionTimeMillis).onBothProvided(_ + _), - None))) + private def runBaselineQuery( + runner: QueryRunner, + session: SparkSession, + desc: String, + id: String, + explain: Boolean): QueryResult = { + val testDesc = "Baseline %s [%s]".format(desc, id) + val result = runner.runQuery(session, testDesc, id, explain = explain) + result } - private[integration] def runQuery( - creator: TableCreator, - id: String, - explain: Boolean, + private def runTestQuery( + runner: QueryRunner, + session: SparkSession, desc: String, - sessionSwitcher: SparkSessionSwitcher, - runner: QueryRunner): TestResultLine = { - println(s"Running query: $id...") - try { - val baseLineDesc = "Vanilla Spark %s %s".format(desc, id) - sessionSwitcher.useSession("baseline", baseLineDesc) - runner.createTables(creator, sessionSwitcher.spark()) - val expected = - runner.runQuery(sessionSwitcher.spark(), baseLineDesc, id, explain = explain) - val expectedRows = expected.rows - val testDesc = "Gluten Spark %s %s".format(desc, id) - sessionSwitcher.useSession("test", testDesc) - runner.createTables(creator, sessionSwitcher.spark()) - val result = runner.runQuery(sessionSwitcher.spark(), testDesc, id, explain = explain) - val resultRows = result.rows - val error = TestUtils.compareAnswers(resultRows, expectedRows, sort = true) - if (error.isEmpty) { - println( - s"Successfully ran query $id, result check was passed. " + - s"Returned row count: ${resultRows.length}, expected: ${expectedRows.length}") - return TestResultLine( - id, - testPassed = true, - Some(expectedRows.length), - Some(resultRows.length), - Some(expected.planningTimeMillis), - Some(result.planningTimeMillis), - Some(expected.executionTimeMillis), - Some(result.executionTimeMillis), - None) - } - println(s"Error running query $id, result check was not passed. " + - s"Returned row count: ${resultRows.length}, expected: ${expectedRows.length}, error: ${error.get}") - TestResultLine( - id, - testPassed = false, - Some(expectedRows.length), - Some(resultRows.length), - Some(expected.planningTimeMillis), - Some(result.planningTimeMillis), - Some(expected.executionTimeMillis), - Some(result.executionTimeMillis), - error) - } catch { - case e: Exception => - val error = Some(s"FATAL: ${ExceptionUtils.getStackTrace(e)}") - println( - s"Error running query $id. " + - s" Error: ${error.get}") - TestResultLine(id, testPassed = false, None, None, None, None, None, None, error) - } + id: String, + explain: Boolean): QueryResult = { + val testDesc = "Query %s [%s]".format(desc, id) + val result = runner.runQuery(session, testDesc, id, explain = explain) + result } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala index 76f43cb71b35..1742b99c246d 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/SparkShell.scala @@ -21,7 +21,7 @@ import org.apache.spark.repl.Main case class SparkShell(scale: Double, genPartitionedData: Boolean) extends Action { override def execute(suite: Suite): Boolean = { - suite.sessionSwitcher.useSession("test", "Gluten Spark CLI") + suite.sessionSwitcher.useSession("test", "Spark CLI") val runner: QueryRunner = new QueryRunner(suite.queryResource(), suite.dataWritePath(scale, genPartitionedData)) runner.createTables(suite.tableCreator(), suite.sessionSwitcher.spark()) diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala index 6046ae4aaa35..a84915ebe1a9 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/action/package.scala @@ -17,13 +17,31 @@ package org.apache.gluten.integration +import org.apache.spark.sql.RunResult + package object action { - implicit class DualOptionsOps[T](value: (Option[T], Option[T])) { - def onBothProvided[R](func: (T, T) => R): Option[R] = { - if (value._1.isEmpty || value._2.isEmpty) { - return None + + implicit class QueryResultsOps(results: Iterable[QueryRunner.QueryResult]) { + def asSuccesses(): Iterable[QueryRunner.Success] = { + results.map(_.asSuccess()) + } + + def asFailures(): Iterable[QueryRunner.Failure] = { + results.map(_.asFailure()) + } + } + + implicit class CompletedOps(completed: Iterable[QueryRunner.Success]) { + def agg(name: String): Option[QueryRunner.Success] = { + completed.reduceOption { (c1, c2) => + QueryRunner.Success( + name, + RunResult( + c1.runResult.rows ++ c2.runResult.rows, + c1.runResult.planningTimeMillis + c2.runResult.planningTimeMillis, + c1.runResult.executionTimeMillis + c2.runResult.executionTimeMillis, + (c1.runResult.metrics, c2.runResult.metrics).sumUp)) } - Some(func(value._1.get, value._2.get)) } } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala index ba772f165190..add7b01feb6c 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/clickbench/ClickBenchDataGen.scala @@ -31,7 +31,10 @@ class ClickBenchDataGen(val spark: SparkSession, dir: String) extends DataGen { // Directly download from official URL. val target = new File(dir + File.separator + FILE_NAME) FileUtils.forceMkdirParent(target) - val code = Process(s"wget -P $dir $DATA_URL") !; + val cmd = + s"wget --no-verbose --show-progress --progress=bar:force:noscroll -O $target $DATA_URL" + println(s"Executing command: $cmd") + val code = Process(cmd) !; if (code != 0) { throw new RuntimeException("Download failed") } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkSessionSwitcher.scala b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkSessionSwitcher.scala index 17a50fd295a7..0a1a25351ad0 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkSessionSwitcher.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/spark/sql/SparkSessionSwitcher.scala @@ -65,6 +65,16 @@ class SparkSessionSwitcher(val masterUrl: String, val logLevel: String) extends useSession(SessionDesc(SessionToken(token), appName)) } + def renewSession(): Unit = synchronized { + if (!hasActiveSession()) { + return + } + val sd = _activeSessionDesc + println(s"Renewing $sd session... ") + stopActiveSession() + useSession(sd) + } + private def useSession(desc: SessionDesc): Unit = synchronized { if (desc == _activeSessionDesc) { return From 4c7d8becfcfb4abf544d3ed75b9d5ee44309fe93 Mon Sep 17 00:00:00 2001 From: Jacky Lee Date: Wed, 19 Jun 2024 13:35:42 +0800 Subject: [PATCH 295/402] [VL] Support linking system libprotobuf.a when building arrow (#6129) --- ep/build-velox/src/modify_arrow.patch | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ep/build-velox/src/modify_arrow.patch b/ep/build-velox/src/modify_arrow.patch index 5814958a936f..7d4d8e557b58 100644 --- a/ep/build-velox/src/modify_arrow.patch +++ b/ep/build-velox/src/modify_arrow.patch @@ -1,3 +1,42 @@ +diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt +index d56f6a36d..9b4088df9 100644 +--- a/cpp/CMakeLists.txt ++++ b/cpp/CMakeLists.txt +@@ -773,8 +773,7 @@ if(ARROW_ORC) + list(APPEND ARROW_SHARED_LINK_LIBS orc::orc ${ARROW_PROTOBUF_LIBPROTOBUF}) + list(APPEND ARROW_STATIC_LINK_LIBS orc::orc ${ARROW_PROTOBUF_LIBPROTOBUF}) + if(ORC_SOURCE STREQUAL "SYSTEM") +- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::orc +- ${ARROW_PROTOBUF_LIBPROTOBUF}) ++ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::orc) + endif() + endif() + +@@ -823,9 +822,6 @@ if(ARROW_WITH_OPENTELEMETRY) + opentelemetry-cpp::ostream_span_exporter + opentelemetry-cpp::otlp_http_exporter) + endif() +- if(Protobuf_SOURCE STREQUAL "SYSTEM") +- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_PROTOBUF_LIBPROTOBUF}) +- endif() + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl) + endif() + +@@ -860,6 +856,14 @@ if(ARROW_USE_XSIMD) + list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_XSIMD}) + endif() + ++# This should be done after if(ARROW_ORC) and if(ARROW_WITH_OPENTELEMETRY) ++# because they depend on Protobuf. ++if(ARROW_WITH_PROTOBUF) ++ if(Protobuf_SOURCE STREQUAL "SYSTEM") ++ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_PROTOBUF_LIBPROTOBUF}) ++ endif() ++endif() ++ + add_custom_target(arrow_dependencies) + add_custom_target(arrow_benchmark_dependencies) + add_custom_target(arrow_test_dependencies) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a2627c190..e453512e6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake From 91c9713c9b4ae5e2b0eedad6622ea4310dd594a5 Mon Sep 17 00:00:00 2001 From: Jacky Lee Date: Wed, 19 Jun 2024 13:40:07 +0800 Subject: [PATCH 296/402] [MINOR] Add spark 3.4.x and 3.5.x options in Github issue --- .github/ISSUE_TEMPLATE/bug.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index ad2667ed87da..1de2d1c755de 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -50,6 +50,8 @@ body: options: - Spark-3.2.x - Spark-3.3.x + - Spark-3.4.x + - Spark-3.5.x validations: required: false From 02e96f9edee164542a4600d7ffb04df5d4ccaecf Mon Sep 17 00:00:00 2001 From: James Xu Date: Wed, 19 Jun 2024 13:41:41 +0800 Subject: [PATCH 297/402] [GLUTEN-6134] Polish Configuration.md (#6135) --- docs/Configuration.md | 53 ++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index a148ec3aa4cb..089675286f68 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -22,9 +22,9 @@ You can add these configurations into spark-defaults.conf to enable or disable t | spark.plugins | To load Gluten's components by Spark's plug-in loader | org.apache.gluten.GlutenPlugin | | spark.shuffle.manager | To turn on Gluten Columnar Shuffle Plugin | org.apache.spark.shuffle.sort.ColumnarShuffleManager | | spark.gluten.enabled | Enable Gluten, default is true. Just an experimental property. Recommend to enable/disable Gluten through the setting for `spark.plugins`. | true | +| spark.gluten.memory.isolation | (Experimental) Enable isolated memory mode. If true, Gluten controls the maximum off-heap memory can be used by each task to X, X = executor memory / max task slots. It's recommended to set true if Gluten serves concurrent queries within a single session, since not all memory Gluten allocated is guaranteed to be spillable. In the case, the feature should be enabled to avoid OOM. Note when true, setting spark.memory.storageFraction to a lower value is suggested since storage memory is considered non-usable by Gluten. | false | | spark.gluten.ras.enabled | Experimental: Enables RAS (relation algebra selector) during physical planning to generate more efficient query plan. Note, this feature is still in development and may not bring performance profits. | false | | spark.gluten.sql.columnar.maxBatchSize | Number of rows to be processed in each batch. Default value is 4096. | 4096 | -| spark.gluten.memory.isolation | (Experimental) Enable isolated memory mode. If true, Gluten controls the maximum off-heap memory can be used by each task to X, X = executor memory / max task slots. It's recommended to set true if Gluten serves concurrent queries within a single session, since not all memory Gluten allocated is guaranteed to be spillable. In the case, the feature should be enabled to avoid OOM. Note when true, setting spark.memory.storageFraction to a lower value is suggested since storage memory is considered non-usable by Gluten. | false | | spark.gluten.sql.columnar.scanOnly | When enabled, this config will overwrite all other operators' enabling, and only Scan and Filter pushdown will be offloaded to native. | false | | spark.gluten.sql.columnar.batchscan | Enable or Disable Columnar BatchScan, default is true | true | | spark.gluten.sql.columnar.hashagg | Enable or Disable Columnar Hash Aggregate, default is true | true | @@ -42,7 +42,7 @@ You can add these configurations into spark-defaults.conf to enable or disable t | spark.gluten.sql.columnar.tableCache | Enable or Disable Columnar Table Cache, default is false | true | | spark.gluten.sql.columnar.broadcastExchange | Enable or Disable Columnar Broadcast Exchange, default is true | true | | spark.gluten.sql.columnar.broadcastJoin | Enable or Disable Columnar BroadcastHashJoin, default is true | true | -| spark.gluten.sql.columnar.shuffle.sort.threshold | The threshold to determine whether to use sort-based columnar shuffle. Sort-based shuffle will be used if the number of partitions is greater than this threshold. | 100000 | +| spark.gluten.sql.columnar.shuffle.sort.threshold | The threshold to determine whether to use sort-based columnar shuffle. Sort-based shuffle will be used if the number of partitions is greater than this threshold. | 100000 | | spark.gluten.sql.columnar.shuffle.codec | Set up the codec to be used for Columnar Shuffle. If this configuration is not set, will check the value of spark.io.compression.codec. By default, Gluten use software compression. Valid options for software compression are lz4, zstd. Valid options for QAT and IAA is gzip. | lz4 | | spark.gluten.sql.columnar.shuffle.codecBackend | Enable using hardware accelerators for shuffle de/compression. Valid options are QAT and IAA. | | | spark.gluten.sql.columnar.shuffle.compressionMode | Setting different compression mode in shuffle, Valid options are buffer and rowvector, buffer option compress each buffer of RowVector individually into one pre-allocated large buffer, rowvector option first copies each buffer of RowVector to a large buffer and then compress the entire buffer in one go. | buffer | @@ -51,26 +51,25 @@ You can add these configurations into spark-defaults.conf to enable or disable t | spark.gluten.sql.columnar.shuffle.merge.threshold | Set the threshold control the minimum merged size. When a partition buffer is full, and the number of rows is below (`threshold * spark.gluten.sql.columnar.maxBatchSize`), it will be saved for merging. | 0.25 | | spark.gluten.sql.columnar.numaBinding | Set up NUMABinding, default is false | true | | spark.gluten.sql.columnar.coreRange | Set up the core range for NUMABinding, only works when numaBinding set to true.
The setting is based on the number of cores in your system. Use 72 cores as an example. | 0-17,36-53 |18-35,54-71 | -| spark.gluten.sql.native.bloomFilter | Enable or Disable native runtime bloom filter. | true | -| spark.gluten.sql.native.arrow.reader.enabled | Enable or Disable native arrow read CSV file format | false | | spark.gluten.sql.columnar.wholeStage.fallback.threshold | Configure the threshold for whether whole stage will fall back in AQE supported case by counting the number of ColumnarToRow & vanilla leaf node | \>= 1 | | spark.gluten.sql.columnar.query.fallback.threshold | Configure the threshold for whether query will fall back by counting the number of ColumnarToRow & vanilla leaf node | \>= 1 | | spark.gluten.sql.columnar.fallback.ignoreRowToColumnar | When true, the fallback policy ignores the RowToColumnar when counting fallback number. | true | | spark.gluten.sql.columnar.fallback.preferColumnar | When true, the fallback policy prefers to use Gluten plan rather than vanilla Spark plan if the both of them contains ColumnarToRow and the vanilla Spark plan ColumnarToRow number is not smaller than Gluten plan. | true | -| spark.gluten.sql.columnar.maxBatchSize | Set the number of rows for the output batch. | 4096 | +| spark.gluten.sql.columnar.force.hashagg | Force to use hash agg to replace sort agg. | true | +| spark.gluten.sql.columnar.vanillaReaders | Enable vanilla spark's vectorized reader. Please note it may bring perf. overhead due to extra data transition. We recommend to disable it if most queries can be fully offloaded to gluten. | false | +| spark.gluten.sql.native.bloomFilter | Enable or Disable native runtime bloom filter. | true | +| spark.gluten.sql.native.arrow.reader.enabled | Enable or Disable native arrow read CSV file format | false | | spark.gluten.shuffleWriter.bufferSize | Set the number of buffer rows for the shuffle writer | value of spark.gluten.sql.columnar.maxBatchSize | | spark.gluten.loadLibFromJar | Controls whether to load dynamic link library from a packed jar for gluten/cpp. Not applicable to static build and clickhouse backend. | false | | spark.gluten.loadLibOS | When `spark.gluten.loadLibFromJar` is true. Manually specify the system os to load library, e.g., CentOS | | | spark.gluten.loadLibOSVersion | Manually specify the system os version to load library, e.g., if `spark.gluten.loadLibOS` is CentOS, this config can be 7 | | -| spark.gluten.sql.columnar.force.hashagg | Force to use hash agg to replace sort agg. | true | -| spark.gluten.sql.columnar.vanillaReaders | Enable vanilla spark's vectorized reader. Please note it may bring perf. overhead due to extra data transition. We recommend to disable it if most queries can be fully offloaded to gluten. | false | | spark.gluten.expression.blacklist | A black list of expression to skip transform, multiple values separated by commas. | | | spark.gluten.sql.columnar.fallback.expressions.threshold | Fall back filter/project if the height of expression tree reaches this threshold, considering Spark codegen can bring better performance for such case. | 50 | | spark.gluten.sql.cartesianProductTransformerEnabled | Config to enable CartesianProductExecTransformer. | true | - | spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled | Config to enable BroadcastNestedLoopJoinExecTransformer. | true | - | spark.gluten.sql.cacheWholeStageTransformerContext | When true, `WholeStageTransformer` will cache the `WholeStageTransformerContext` when executing. It is used to get substrait plan node and native plan string. | false | - | spark.gluten.sql.injectNativePlanStringToExplain | When true, Gluten will inject native plan tree to explain string inside `WholeStageTransformerContext`. | false | - | spark.gluten.sql.fallbackRegexpExpressions | When true, Gluten will fall back all regexp expressions to avoid any incompatibility risk. | false | +| spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled | Config to enable BroadcastNestedLoopJoinExecTransformer. | true | +| spark.gluten.sql.cacheWholeStageTransformerContext | When true, `WholeStageTransformer` will cache the `WholeStageTransformerContext` when executing. It is used to get substrait plan node and native plan string. | false | +| spark.gluten.sql.injectNativePlanStringToExplain | When true, Gluten will inject native plan tree to explain string inside `WholeStageTransformerContext`. | false | +| spark.gluten.sql.fallbackRegexpExpressions | When true, Gluten will fall back all regexp expressions to avoid any incompatibility risk. | false | ## Velox Parameters @@ -81,11 +80,11 @@ The following configurations are related to Velox settings. | spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems | The default number of expected items for the velox bloomfilter. | 1000000L | | spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits | The default number of bits to use for the velox bloom filter. | 8388608L | | spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits | The max number of bits to use for the velox bloom filter. | 4194304L | - | spark.gluten.sql.columnar.backend.velox.fileHandleCacheEnabled | Disables caching if false. File handle cache should be disabled if files are mutable, i.e. file content may change while file path stays the same. | | - | spark.gluten.sql.columnar.backend.velox.directorySizeGuess | Set the directory size guess for velox file scan. | | - | spark.gluten.sql.columnar.backend.velox.filePreloadThreshold | Set the file preload threshold for velox file scan. | | - | spark.gluten.sql.columnar.backend.velox.prefetchRowGroups | Set the prefetch row groups for velox file scan. | | - | spark.gluten.sql.columnar.backend.velox.loadQuantum | Set the load quantum for velox file scan. | | +| spark.gluten.sql.columnar.backend.velox.fileHandleCacheEnabled | Disables caching if false. File handle cache should be disabled if files are mutable, i.e. file content may change while file path stays the same. | | +| spark.gluten.sql.columnar.backend.velox.directorySizeGuess | Set the directory size guess for velox file scan. | | +| spark.gluten.sql.columnar.backend.velox.filePreloadThreshold | Set the file preload threshold for velox file scan. | | +| spark.gluten.sql.columnar.backend.velox.prefetchRowGroups | Set the prefetch row groups for velox file scan. | | +| spark.gluten.sql.columnar.backend.velox.loadQuantum | Set the load quantum for velox file scan. | | | spark.gluten.sql.columnar.backend.velox.maxCoalescedDistanceBytes | Set the max coalesced distance bytes for velox file scan. | | | spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes | Set the max coalesced bytes for velox file scan. | | | spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct | Set prefetch cache min pct for velox file scan. | | @@ -94,28 +93,16 @@ The following configurations are related to Velox settings. | spark.gluten.sql.complexType.scan.fallback.enabled | Force fallback for complex type scan, including struct, map, array. | true | -``` -##### Columnar Process Configuration - -spark.plugins org.apache.gluten.GlutenPlugin -spark.shuffle.manager org.apache.spark.shuffle.sort.ColumnarShuffleManager -spark.driver.extraClassPath ${GLUTEN_HOME}/package/target/gluten-XXX.jar -spark.executor.extraClassPath ${GLUTEN_HOME}/package/target/gluten-XXX.jar -###### -``` - Additionally, you can control the configurations of gluten at thread level by local property. -| Parameters | Description | Recommend Setting | -|---------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------| -| gluten.enabledForCurrentThread | Control the usage of gluten at thread level. | true | +| Parameters | Description | Recommend Setting | +|--------------------------------|----------------------------------------------|-------------------| +| gluten.enabledForCurrentThread | Control the usage of gluten at thread level. | true | Below is an example of developing an application using scala to set local properties. -``` -##### Before executing the query, set local properties - +```scala +// Before executing the query, set local properties. sparkContext.setLocalProperty(key, value) spark.sql("select * from demo_tables").show() -###### ``` From f3370bbcc311f5777582c202c49362d93115262c Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Wed, 19 Jun 2024 13:45:51 +0800 Subject: [PATCH 298/402] [CH] support function rint (#6121) [CH] support function rint --------- Co-authored-by: liuneng1994 --- .../gluten/utils/CHExpressionUtil.scala | 1 - .../Functions/SparkFunctionRint.cpp | 64 +++++++++++++++++++ .../Functions/SparkFunctionRint.h | 57 +++++++++++++++++ .../Parser/SerializedPlanParser.h | 2 + .../clickhouse/ClickHouseTestSettings.scala | 1 - .../clickhouse/ClickHouseTestSettings.scala | 1 - 6 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 cpp-ch/local-engine/Functions/SparkFunctionRint.cpp create mode 100644 cpp-ch/local-engine/Functions/SparkFunctionRint.h diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index d47523c0f6ac..cf45c1118f13 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -210,7 +210,6 @@ object CHExpressionUtil { TIMESTAMP_MILLIS -> DefaultValidator(), TIMESTAMP_MICROS -> DefaultValidator(), FLATTEN -> DefaultValidator(), - RINT -> DefaultValidator(), STACK -> DefaultValidator() ) } diff --git a/cpp-ch/local-engine/Functions/SparkFunctionRint.cpp b/cpp-ch/local-engine/Functions/SparkFunctionRint.cpp new file mode 100644 index 000000000000..656994c3ea62 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkFunctionRint.cpp @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SparkFunctionRint.h" + +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} +} + +namespace local_engine +{ + +DB::ColumnPtr SparkFunctionRint::executeImpl( + const DB::ColumnsWithTypeAndName & arguments, + const DB::DataTypePtr & result_type, + size_t /*input_rows_count*/) const +{ + if (arguments.size() != 1) + throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly one argument.", getName()); + if (!isFloat(*arguments[0].type)) + throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument for function {} must be float32 or float64, got {}", getName(), arguments[0].type->getName()); + + auto output = result_type->createColumn(); + bool is_float32 = DB::WhichDataType(*arguments[0].type).isFloat32(); + auto input = arguments[0].column->convertToFullIfNeeded(); + auto& output_data = static_cast(output.get())->getData(); + output_data.resize(input->size()); + for (size_t i = 0; i < input->size(); ++i) + { + if (is_float32) + output_data[i] = std::rint(DB::checkAndGetColumn(*input).getData()[i]); + else + output_data[i] = std::rint(DB::checkAndGetColumn(*input).getData()[i]); + } + return std::move(output); +} + + +REGISTER_FUNCTION(SparkFunctionRint) +{ + factory.registerFunction(); +} +} \ No newline at end of file diff --git a/cpp-ch/local-engine/Functions/SparkFunctionRint.h b/cpp-ch/local-engine/Functions/SparkFunctionRint.h new file mode 100644 index 000000000000..ee7a7bb1ea66 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkFunctionRint.h @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +namespace local_engine +{ +class SparkFunctionRint : public DB::IFunction +{ +public: + static constexpr auto name = "sparkRint"; + + static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared(); } + + SparkFunctionRint() = default; + + ~SparkFunctionRint() override = default; + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + bool isSuitableForShortCircuitArgumentsExecution(const DB::DataTypesWithConstInfo &) const override { return true; } + + DB::ColumnPtr executeImpl( + const DB::ColumnsWithTypeAndName & arguments, + const DB::DataTypePtr & result_type, + size_t input_rows_count) const override; + + DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes &) const override + { + return std::make_shared(); + } +}; +} diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index c79598c59923..ccd5c0fdc4c8 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -112,6 +112,7 @@ static const std::map SCALAR_FUNCTIONS {"rand", "randCanonical"}, {"isnan", "isNaN"}, {"bin", "sparkBin"}, + {"rint", "sparkRint"}, /// string functions {"like", "like"}, @@ -150,6 +151,7 @@ static const std::map SCALAR_FUNCTIONS {"initcap", "initcapUTF8"}, {"conv", "sparkConv"}, {"uuid", "generateUUIDv4"}, + {"levenshteinDistance", "editDistanceUTF8"}, /// hash functions {"crc32", "CRC32"}, diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index a8a5a1e412a8..19c9b2cf478f 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -844,7 +844,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .excludeGlutenTest("default") enableSuite[GlutenMathExpressionsSuite] .exclude("tanh") - .exclude("rint") .exclude("unhex") .exclude("atan2") .exclude("round/bround") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index cb33f002553b..da71110de3b4 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -849,7 +849,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-37967: Literal.create support ObjectType") enableSuite[GlutenMathExpressionsSuite] .exclude("tanh") - .exclude("rint") .exclude("unhex") .exclude("atan2") .exclude("round/bround/floor/ceil") From 7717f7f74c67b4d8ff7e43b2495d7ee7c707ca11 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:38:31 +0800 Subject: [PATCH 299/402] [VL] Daily Update Velox Version (2024_06_19) (#6138) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 8d5bd8d167fe..26374c2be504 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_18 +VELOX_BRANCH=2024_06_19 VELOX_HOME="" #Set on run gluten on HDFS From d7b978d6dd469e18599179f7136208471994a936 Mon Sep 17 00:00:00 2001 From: deepa <8832194+deepashreeraghu@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:21:58 +0530 Subject: [PATCH 300/402] [GLUTEN-6064][VL] Support loading shared libraries on RedHat-9 (#6063) --- .../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 0ad267d4c8d3..41b56804b50b 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -80,6 +80,8 @@ class VeloxListenerApi extends ListenerApi { new SharedLibraryLoaderCentos7 } else if (system.contains("tencentos") && system.contains("3.2")) { new SharedLibraryLoaderCentos8 + } else if (systemName.contains("Red Hat") && systemVersion.startsWith("9")) { + new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("8")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("7")) { @@ -92,7 +94,7 @@ class VeloxListenerApi extends ListenerApi { throw new GlutenException( s"Found unsupported OS($systemName, $systemVersion)! Currently, Gluten's Velox backend" + " only supports Ubuntu 20.04/22.04, CentOS 7/8, " + - "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 3.2, RedHat 7/8, " + + "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 3.2, RedHat 7/8/9, " + "Debian 11/12.") } } From 10471954a2cf944ba082623c6dd66da62a4deeef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Wed, 19 Jun 2024 14:57:22 +0800 Subject: [PATCH 301/402] [GLUTEN-6016][CH] Add uts for decimal convert to int overflow case (#6018) * add uts for decimal convert to int overflow case * add uts for decimal convert to int overflow case * add uts for decimal convert to int overflow case --- .../gluten/execution/GlutenClickHouseDecimalSuite.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala index 892d2ff61855..088487101081 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala @@ -443,6 +443,11 @@ class GlutenClickHouseDecimalSuite ) } + test("Fix issue(6015) allow overflow when converting decimal to integer") { + val sql = "select int(cast(id * 9999999999 as decimal(29, 2))) from range(10)" + runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer]) + } + def testFromRandomBase( sql: String, customCheck: DataFrame => Unit, From 99596c9ccdcbfd7c4b4f0edd65632a01b6d24b7e Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 19 Jun 2024 15:47:48 +0800 Subject: [PATCH 302/402] [VL] Minor command script correction in GHA CI --- .github/workflows/velox_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index b1d5cfdcf3d3..f690ac7aedc4 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -461,7 +461,7 @@ jobs: run: | export MAVEN_HOME=/usr/lib/maven && \ export PATH=${PATH}:${MAVEN_HOME}/bin && \ - export export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ + export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ cd /opt && \ git clone -b branch-0.8 https://github.com/apache/incubator-uniffle.git && \ cd incubator-uniffle && \ From 27c32f1b15d42a8cf98b20b0cb67c9de0c0edb7e Mon Sep 17 00:00:00 2001 From: Kaifei Yi Date: Thu, 20 Jun 2024 00:06:01 +0800 Subject: [PATCH 303/402] [VL] Prefer to use path.getFileSystem instead of FileSystem.get to create FileSystem (#6123) Co-authored-by: yikaifei --- .../datasources/velox/VeloxFormatWriterInjects.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala index c358d6372c36..9f9d4332640c 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala @@ -34,7 +34,7 @@ import org.apache.spark.util.TaskResources import com.google.common.base.Preconditions import org.apache.arrow.c.ArrowSchema -import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} +import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hadoop.mapreduce.TaskAttemptContext import java.io.IOException @@ -48,8 +48,8 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase { // Create the hdfs path if not existed. val hdfsSchema = "hdfs://" if (filePath.startsWith(hdfsSchema)) { - val fs = FileSystem.get(context.getConfiguration) val hdfsPath = new Path(filePath) + val fs = hdfsPath.getFileSystem(context.getConfiguration) if (!fs.exists(hdfsPath.getParent)) { fs.mkdirs(hdfsPath.getParent) } From 716dd9a8ba92d46e70405483231c767f4ccf9259 Mon Sep 17 00:00:00 2001 From: Ankita Victor Date: Thu, 20 Jun 2024 07:37:21 +0530 Subject: [PATCH 304/402] Revert SortShuffleManager changes in ColumnarShuffleManager (#6149) --- .../shuffle/sort/ColumnarShuffleManager.scala | 121 ++++++++++++------ 1 file changed, 79 insertions(+), 42 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala b/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala index 06c6e6c0ea5a..d8ba78cb98fd 100644 --- a/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala +++ b/gluten-core/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala @@ -20,6 +20,7 @@ import org.apache.spark.{ShuffleDependency, SparkConf, SparkEnv, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.serializer.SerializerManager import org.apache.spark.shuffle._ +import org.apache.spark.shuffle.api.ShuffleExecutorComponents import org.apache.spark.shuffle.sort.SortShuffleManager.canUseBatchFetch import org.apache.spark.storage.BlockId import org.apache.spark.util.collection.OpenHashSet @@ -27,12 +28,13 @@ import org.apache.spark.util.collection.OpenHashSet import java.io.InputStream import java.util.concurrent.ConcurrentHashMap +import scala.collection.JavaConverters._ + class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { import ColumnarShuffleManager._ - private[this] lazy val sortShuffleManager: SortShuffleManager = new SortShuffleManager(conf) - + private lazy val shuffleExecutorComponents = loadShuffleExecutorComponents(conf) override val shuffleBlockResolver = new IndexShuffleBlockResolver(conf) /** A mapping from shuffle ids to the number of mappers producing output for those shuffles. */ @@ -47,9 +49,23 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin new ColumnarShuffleHandle[K, V]( shuffleId, dependency.asInstanceOf[ColumnarShuffleDependency[K, V, V]]) + } else if (SortShuffleWriter.shouldBypassMergeSort(conf, dependency)) { + // If there are fewer than spark.shuffle.sort.bypassMergeThreshold partitions and we don't + // need map-side aggregation, then write numPartitions files directly and just concatenate + // them at the end. This avoids doing serialization and deserialization twice to merge + // together the spilled files, which would happen with the normal code path. The downside is + // having multiple files open at a time and thus more memory allocated to buffers. + new BypassMergeSortShuffleHandle[K, V]( + shuffleId, + dependency.asInstanceOf[ShuffleDependency[K, V, V]]) + } else if (SortShuffleManager.canUseSerializedShuffle(dependency)) { + // Otherwise, try to buffer map outputs in a serialized form, since this is more efficient: + new SerializedShuffleHandle[K, V]( + shuffleId, + dependency.asInstanceOf[ShuffleDependency[K, V, V]]) } else { - // Otherwise call default SortShuffleManager - sortShuffleManager.registerShuffle(shuffleId, dependency) + // Otherwise, buffer map outputs in a deserialized form: + new BaseShuffleHandle(shuffleId, dependency) } } @@ -59,19 +75,39 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin mapId: Long, context: TaskContext, metrics: ShuffleWriteMetricsReporter): ShuffleWriter[K, V] = { + val mapTaskIds = + taskIdMapsForShuffle.computeIfAbsent(handle.shuffleId, _ => new OpenHashSet[Long](16)) + mapTaskIds.synchronized { + mapTaskIds.add(context.taskAttemptId()) + } + val env = SparkEnv.get handle match { case columnarShuffleHandle: ColumnarShuffleHandle[K @unchecked, V @unchecked] => - val mapTaskIds = - taskIdMapsForShuffle.computeIfAbsent(handle.shuffleId, _ => new OpenHashSet[Long](16)) - mapTaskIds.synchronized { - mapTaskIds.add(context.taskAttemptId()) - } GlutenShuffleWriterWrapper.genColumnarShuffleWriter( shuffleBlockResolver, columnarShuffleHandle, mapId, metrics) - case _ => sortShuffleManager.getWriter(handle, mapId, context, metrics) + case unsafeShuffleHandle: SerializedShuffleHandle[K @unchecked, V @unchecked] => + new UnsafeShuffleWriter( + env.blockManager, + context.taskMemoryManager(), + unsafeShuffleHandle, + mapId, + context, + env.conf, + metrics, + shuffleExecutorComponents) + case bypassMergeSortHandle: BypassMergeSortShuffleHandle[K @unchecked, V @unchecked] => + new BypassMergeSortShuffleWriter( + env.blockManager, + bypassMergeSortHandle, + mapId, + env.conf, + metrics, + shuffleExecutorComponents) + case other: BaseShuffleHandle[K @unchecked, V @unchecked, _] => + new SortShuffleWriter(other, mapId, context, shuffleExecutorComponents) } } @@ -87,17 +123,17 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin endPartition: Int, context: TaskContext, metrics: ShuffleReadMetricsReporter): ShuffleReader[K, C] = { + val (blocksByAddress, canEnableBatchFetch) = { + GlutenShuffleUtils.getReaderParam( + handle, + startMapIndex, + endMapIndex, + startPartition, + endPartition) + } + val shouldBatchFetch = + canEnableBatchFetch && canUseBatchFetch(startPartition, endPartition, context) if (handle.isInstanceOf[ColumnarShuffleHandle[_, _]]) { - val (blocksByAddress, canEnableBatchFetch) = { - GlutenShuffleUtils.getReaderParam( - handle, - startMapIndex, - endMapIndex, - startPartition, - endPartition) - } - val shouldBatchFetch = - canEnableBatchFetch && canUseBatchFetch(startPartition, endPartition, context) new BlockStoreShuffleReader( handle.asInstanceOf[BaseShuffleHandle[K, _, C]], blocksByAddress, @@ -107,43 +143,44 @@ class ColumnarShuffleManager(conf: SparkConf) extends ShuffleManager with Loggin shouldBatchFetch = shouldBatchFetch ) } else { - sortShuffleManager.getReader( - handle, - startMapIndex, - endMapIndex, - startPartition, - endPartition, + new BlockStoreShuffleReader( + handle.asInstanceOf[BaseShuffleHandle[K, _, C]], + blocksByAddress, context, - metrics) + metrics, + shouldBatchFetch = shouldBatchFetch + ) } } /** Remove a shuffle's metadata from the ShuffleManager. */ override def unregisterShuffle(shuffleId: Int): Boolean = { - if (taskIdMapsForShuffle.contains(shuffleId)) { - Option(taskIdMapsForShuffle.remove(shuffleId)).foreach { - mapTaskIds => - mapTaskIds.iterator.foreach { - mapId => shuffleBlockResolver.removeDataByMap(shuffleId, mapId) - } - } - true - } else { - sortShuffleManager.unregisterShuffle(shuffleId) + Option(taskIdMapsForShuffle.remove(shuffleId)).foreach { + mapTaskIds => + mapTaskIds.iterator.foreach { + mapId => shuffleBlockResolver.removeDataByMap(shuffleId, mapId) + } } + true } /** Shut down this ShuffleManager. */ override def stop(): Unit = { - if (!taskIdMapsForShuffle.isEmpty) { - shuffleBlockResolver.stop() - } else { - sortShuffleManager.stop - } + shuffleBlockResolver.stop() } } object ColumnarShuffleManager extends Logging { + private def loadShuffleExecutorComponents(conf: SparkConf): ShuffleExecutorComponents = { + val executorComponents = ShuffleDataIOUtils.loadShuffleDataIO(conf).executor() + val extraConfigs = conf.getAllWithPrefix(ShuffleDataIOUtils.SHUFFLE_SPARK_CONF_PREFIX).toMap + executorComponents.initializeExecutor( + conf.getAppId, + SparkEnv.get.executorId, + extraConfigs.asJava) + executorComponents + } + private def bypassDecompressionSerializerManger = new SerializerManager( SparkEnv.get.serializer, From bfd089ecca22365d50592de5326f662a55c3d463 Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Thu, 20 Jun 2024 07:46:56 +0530 Subject: [PATCH 305/402] [CORE] Support JDK 11 (#6112) --- .github/workflows/velox_docker.yml | 32 ++++++++++++++++++++++++++---- docs/developers/NewToGluten.md | 2 +- pom.xml | 15 ++++++++++++-- tools/gluten-it/pom.xml | 9 +++++++++ 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index f690ac7aedc4..5f64c9f7e0e8 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -87,7 +87,7 @@ jobs: matrix: os: [ "ubuntu:20.04", "ubuntu:22.04" ] spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ] - java: [ "java-8", "java-17" ] + java: [ "java-8", "java-11", "java-17" ] # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772 exclude: - spark: spark-3.2 @@ -96,8 +96,16 @@ jobs: java: java-17 - spark: spark-3.5 java: java-17 + - spark: spark-3.2 + java: java-11 + - spark: spark-3.3 + java: java-11 + - spark: spark-3.4 + java: java-11 - os: ubuntu:22.04 java: java-17 + - os: ubuntu:22.04 + java: java-11 runs-on: ubuntu-20.04 container: ${{ matrix.os }} steps: @@ -116,10 +124,13 @@ jobs: run: | if [ "${{ matrix.java }}" = "java-17" ]; then apt-get update && apt-get install -y openjdk-17-jdk maven + apt remove openjdk-11* -y + elif [ "${{ matrix.java }}" = "java-11" ]; then + apt-get update && apt-get install -y openjdk-11-jdk maven else apt-get update && apt-get install -y openjdk-8-jdk maven + apt remove openjdk-11* -y fi - apt remove openjdk-11* -y ls -l /root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/ - name: Build and run TPCH/DS run: | @@ -141,7 +152,7 @@ jobs: matrix: os: [ "centos:7", "centos:8" ] spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ] - java: [ "java-8", "java-17" ] + java: [ "java-8", "java-11", "java-17" ] # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772 exclude: - spark: spark-3.2 @@ -150,8 +161,16 @@ jobs: java: java-17 - spark: spark-3.5 java: java-17 + - spark: spark-3.2 + java: java-11 + - spark: spark-3.3 + java: java-11 + - spark: spark-3.4 + java: java-11 - os: centos:7 java: java-17 + - os: centos:7 + java: java-11 runs-on: ubuntu-20.04 container: ${{ matrix.os }} steps: @@ -175,6 +194,8 @@ jobs: run: | if [ "${{ matrix.java }}" = "java-17" ]; then yum update -y && yum install -y java-17-openjdk-devel wget + elif [ "${{ matrix.java }}" = "java-11" ]; then + yum update -y && yum install -y java-11-openjdk-devel wget else yum update -y && yum install -y java-1.8.0-openjdk-devel wget fi @@ -186,6 +207,8 @@ jobs: echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV if [ "${{ matrix.java }}" = "java-17" ]; then echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV + elif [ "${{ matrix.java }}" = "java-11" ]; then + echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk" >> $GITHUB_ENV else echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV fi @@ -510,13 +533,14 @@ jobs: - name: Setup java and maven run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget + apt remove openjdk-11* -y + echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }} run: | - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 EXTRA_PROFILE="" if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then EXTRA_PROFILE="-Pceleborn-0.4" diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index a8862f7a5fa0..fa2ca520f232 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -43,7 +43,7 @@ export PATH="$PATH:$JAVA_HOME/bin" ## OpenJDK 17 -By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17, and please make sure your JAVA_HOME points to jdk17. +By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17 or `-Pjava-11` to use JDK 11, and please make sure your JAVA_HOME points to jdk17 or jdk11 respectively. Apache Spark and Arrow requires setting java args `-Dio.netty.tryReflectionSetAccessible=true`, see [SPARK-29924](https://issues.apache.org/jira/browse/SPARK-29924) and [ARROW-6206](https://issues.apache.org/jira/browse/ARROW-6206). So please add following configs in `spark-defaults.conf`: diff --git a/pom.xml b/pom.xml index 88cbb724e053..87b60e40ea85 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ 1.8 ${java.version} ${java.version} - 2.9.3 + 2.9.3 2.12 2.12.15 3 @@ -122,6 +122,16 @@ 1.8 + + java-11 + + 11 + + + 11 + 3.1.8 + + java-17 @@ -129,6 +139,7 @@ 17 + 3.1.8 @@ -301,7 +312,7 @@ com.github.ben-manes.caffeine caffeine - ${caffeine.version.java8} + ${caffeine.version} org.apache.spark diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index ccb59ade8aa5..3f1760069792 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -119,6 +119,15 @@ 1.8 + + java-11 + + 11 + + + 11 + + java-17 From c290ac7727a8a11620f2e07747fb9b1d23be6ff4 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:03:50 +0800 Subject: [PATCH 306/402] [VL] Daily Update Velox Version (2024_06_19) (#6153) Signed-off-by: glutenperfbot Co-authored-by: glutenperfbot From f12dbef7c41cc57116bc579ebaff1091e6e78c4d Mon Sep 17 00:00:00 2001 From: Jacky Lee Date: Thu, 20 Jun 2024 11:04:44 +0800 Subject: [PATCH 307/402] [VL] Avoid use WriteFilesSpec which is not serialzable (#6144) --- .../VeloxColumnarWriteFilesExec.scala | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala index 1d3d55afb526..c87b8d4f688d 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/VeloxColumnarWriteFilesExec.scala @@ -24,8 +24,8 @@ import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.{Partition, SparkException, TaskContext, TaskOutputFileAlreadyExistException} +import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils} import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage -import org.apache.spark.internal.io.SparkHadoopWriterUtils import org.apache.spark.rdd.RDD import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.sql.catalyst.InternalRow @@ -90,7 +90,8 @@ case class VeloxWriteFilesMetrics( */ class VeloxColumnarWriteFilesRDD( var prev: RDD[ColumnarBatch], - writeFilesSpec: WriteFilesSpec, + description: WriteJobDescription, + committer: FileCommitProtocol, jobTrackerID: String) extends RDD[WriterCommitMessage](prev) { @@ -118,7 +119,7 @@ class VeloxColumnarWriteFilesRDD( val fileWriteInfo = fileWriteInfos.head numBytes += fileWriteInfo.fileSize val targetFileName = fileWriteInfo.targetFileName - val outputPath = writeFilesSpec.description.path + val outputPath = description.path // part1=1/part2=1 val partitionFragment = metrics.name @@ -126,7 +127,7 @@ class VeloxColumnarWriteFilesRDD( if (partitionFragment != "") { updatedPartitions += partitionFragment val tmpOutputPath = outputPath + "/" + partitionFragment + "/" + targetFileName - val customOutputPath = writeFilesSpec.description.customPartitionLocations.get( + val customOutputPath = description.customPartitionLocations.get( PartitioningUtils.parsePathFragment(partitionFragment)) if (customOutputPath.isDefined) { addedAbsPathFiles(tmpOutputPath) = customOutputPath.get + "/" + targetFileName @@ -174,8 +175,6 @@ class VeloxColumnarWriteFilesRDD( private def writeFilesForEmptyIterator( commitProtocol: SparkWriteFilesCommitProtocol): WriteTaskResult = { - val description = writeFilesSpec.description - val committer = writeFilesSpec.committer val taskAttemptContext = commitProtocol.taskAttemptContext val dataWriter = @@ -194,10 +193,7 @@ class VeloxColumnarWriteFilesRDD( } override def compute(split: Partition, context: TaskContext): Iterator[WriterCommitMessage] = { - val commitProtocol = new SparkWriteFilesCommitProtocol( - jobTrackerID, - writeFilesSpec.description, - writeFilesSpec.committer) + val commitProtocol = new SparkWriteFilesCommitProtocol(jobTrackerID, description, committer) commitProtocol.setupTask() val writePath = commitProtocol.newTaskAttemptTempPath() @@ -238,7 +234,7 @@ class VeloxColumnarWriteFilesRDD( case t: Throwable => throw new SparkException( s"Task failed while writing rows to staging path: $writePath, " + - s"output path: ${writeFilesSpec.description.path}", + s"output path: ${description.path}", t) } @@ -285,10 +281,9 @@ case class VeloxColumnarWriteFilesExec private ( /** Fallback to use vanilla Spark write files to generate an empty file for metadata only. */ private def writeFilesForEmptyRDD( - writeFilesSpec: WriteFilesSpec, + description: WriteJobDescription, + committer: FileCommitProtocol, jobTrackerID: String): RDD[WriterCommitMessage] = { - val description = writeFilesSpec.description - val committer = writeFilesSpec.committer val rddWithNonEmptyPartitions = session.sparkContext.parallelize(Seq.empty[InternalRow], 1) rddWithNonEmptyPartitions.mapPartitionsInternal { iterator => @@ -314,12 +309,14 @@ case class VeloxColumnarWriteFilesExec private ( val rdd = child.executeColumnar() val jobTrackerID = SparkHadoopWriterUtils.createJobTrackerID(new Date()) + val description = writeFilesSpec.description + val committer = writeFilesSpec.committer if (rdd.partitions.length == 0) { // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single // partition rdd to make sure we at least set up one write task to write the metadata. - writeFilesForEmptyRDD(writeFilesSpec, jobTrackerID) + writeFilesForEmptyRDD(description, committer, jobTrackerID) } else { - new VeloxColumnarWriteFilesRDD(rdd, writeFilesSpec, jobTrackerID) + new VeloxColumnarWriteFilesRDD(rdd, description, committer, jobTrackerID) } } override protected def withNewChildrenInternal( From 38d7e109bf3daa4af634639d8b4ffc3ac6773b60 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Thu, 20 Jun 2024 11:10:02 +0800 Subject: [PATCH 308/402] [CORE] Add custom cost evaluator for optimize buildSide of shuffled hash join (#6143) --- .../org/apache/gluten/GlutenPlugin.scala | 6 +++ .../org/apache/gluten/GlutenConfig.scala | 15 +++++++ .../sql/execution/adaptive/GlutenCost.scala | 43 +++++++++++++++++++ .../adaptive/GlutenCostEvaluator.scala | 32 ++++++++++++++ .../adaptive/GlutenCostEvaluator.scala | 35 +++++++++++++++ .../adaptive/GlutenCostEvaluator.scala | 35 +++++++++++++++ .../adaptive/GlutenCostEvaluator.scala | 35 +++++++++++++++ 7 files changed, 201 insertions(+) create mode 100644 shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala create mode 100644 shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala create mode 100644 shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala create mode 100644 shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala create mode 100644 shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index cafed66eb8f0..2860e3ced072 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -143,6 +143,12 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { } conf.set(SPARK_SESSION_EXTS_KEY, extensions) + // adaptive custom cost evaluator class + if (GlutenConfig.getConf.enableGluten && GlutenConfig.getConf.enableGlutenCostEvaluator) { + val costEvaluator = "org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator" + conf.set(SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key, costEvaluator) + } + // check memory off-heap enabled and size val minOffHeapSize = "1MB" if ( diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index eb6118ffc74e..462032488548 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -437,6 +437,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { def enableCastAvgAggregateFunction: Boolean = conf.getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED) + def enableGlutenCostEvaluator: Boolean = conf.getConf(COST_EVALUATOR_ENABLED) + def dynamicOffHeapSizingEnabled: Boolean = conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) @@ -595,6 +597,8 @@ object GlutenConfig { val GLUTEN_DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION = "spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction" + val GLUTEN_COST_EVALUATOR_ENABLED = "spark.gluten.sql.adaptive.costEvaluator.enabled" + var ins: GlutenConfig = _ def getConf: GlutenConfig = { @@ -1962,6 +1966,17 @@ object GlutenConfig { .booleanConf .createWithDefault(true) + val COST_EVALUATOR_ENABLED = + buildConf(GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED) + .internal() + .doc( + "If true and gluten enabled, use " + + "org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost " + + "evaluator class, else follow the configuration " + + "spark.sql.adaptive.customCostEvaluatorClass.") + .booleanConf + .createWithDefault(true) + val DYNAMIC_OFFHEAP_SIZING_ENABLED = buildConf(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED) .internal() diff --git a/shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala b/shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala new file mode 100644 index 000000000000..9436f469ba79 --- /dev/null +++ b/shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.adaptive + +import org.apache.spark.sql.errors.QueryExecutionErrors +import org.apache.spark.sql.execution.SparkPlan + +class GlutenCost(val eval: CostEvaluator, val plan: SparkPlan) extends Cost { + override def compare(that: Cost): Int = that match { + case that: GlutenCost if plan eq that.plan => + 0 + case that: GlutenCost if plan == that.plan => + // Plans are identical. Considers the newer one as having lower cost. + -(plan.id - that.plan.id) + case that: GlutenCost => + // Plans are different. Use the delegated cost evaluator. + assert(eval == that.eval) + eval.evaluateCost(plan).compare(eval.evaluateCost(that.plan)) + case _ => + throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString) + } + + override def hashCode(): Int = throw new UnsupportedOperationException() + + override def equals(obj: Any): Boolean = obj match { + case that: Cost => compare(that) == 0 + case _ => false + } +} diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala new file mode 100644 index 000000000000..a6f066462a1a --- /dev/null +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.adaptive + +import org.apache.gluten.GlutenConfig + +import org.apache.spark.sql.execution.SparkPlan + +/** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ +case class GlutenCostEvaluator() extends CostEvaluator { + override def evaluateCost(plan: SparkPlan): Cost = { + if (GlutenConfig.getConf.enableGluten) { + new GlutenCost(SimpleCostEvaluator, plan) + } else { + SimpleCostEvaluator.evaluateCost(plan) + } + } +} diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala new file mode 100644 index 000000000000..8fcfa735f463 --- /dev/null +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.adaptive + +import org.apache.gluten.GlutenConfig + +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.internal.SQLConf + +/** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ +case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { + override def evaluateCost(plan: SparkPlan): Cost = { + val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) + if (GlutenConfig.getConf.enableGluten) { + new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) + } else { + SimpleCostEvaluator(forceOptimizeSkewedJoin).evaluateCost(plan) + } + } +} diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala new file mode 100644 index 000000000000..8fcfa735f463 --- /dev/null +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.adaptive + +import org.apache.gluten.GlutenConfig + +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.internal.SQLConf + +/** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ +case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { + override def evaluateCost(plan: SparkPlan): Cost = { + val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) + if (GlutenConfig.getConf.enableGluten) { + new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) + } else { + SimpleCostEvaluator(forceOptimizeSkewedJoin).evaluateCost(plan) + } + } +} diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala new file mode 100644 index 000000000000..8fcfa735f463 --- /dev/null +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.adaptive + +import org.apache.gluten.GlutenConfig + +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.internal.SQLConf + +/** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ +case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { + override def evaluateCost(plan: SparkPlan): Cost = { + val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) + if (GlutenConfig.getConf.enableGluten) { + new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) + } else { + SimpleCostEvaluator(forceOptimizeSkewedJoin).evaluateCost(plan) + } + } +} From 79e1d588dcad5a655c2e5d363d18f8d695ef1cee Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Thu, 20 Jun 2024 03:04:25 -0500 Subject: [PATCH 309/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240620) (#6150) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240620) * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/61047 * fix style * Using assertResult instead of assert, so we can know the actual result once failed. --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- ...nClickHouseMergeTreeWriteOnHDFSSuite.scala | 111 ++++++++------- ...tenClickHouseMergeTreeWriteOnS3Suite.scala | 127 +++++++++--------- cpp-ch/clickhouse.version | 4 +- .../Mergetree/SparkMergeTreeWriter.cpp | 32 ++--- .../Storages/Mergetree/SparkMergeTreeWriter.h | 13 +- 5 files changed, 141 insertions(+), 146 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala index 56b8f056bc25..572d0cd50a6e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala @@ -57,6 +57,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite .set( "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert", "false") + // .set("spark.gluten.sql.columnar.backend.ch.runtime_config.path", "/data") // for local test } override protected def beforeEach(): Unit = { @@ -139,7 +140,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -151,8 +152,8 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 1) - assert(addFiles.head.rows == 600572) + assertResult(1)(addFiles.size) + assertResult(600572)(addFiles.head.rows) } spark.sql("drop table lineitem_mergetree_hdfs") } @@ -224,7 +225,7 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -232,24 +233,22 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 1) - assert(addFiles.head.rows == 600572) + assertResult(1)(addFiles.size) + assertResult(600572)(addFiles.head.rows) } spark.sql("drop table lineitem_mergetree_orderbykey_hdfs") } @@ -386,51 +385,49 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite runTPCHQueryBySQL(1, sqlStr, compareResult = false) { df => val result = df.collect() - assert(result.length == 4) - assert(result(0).getString(0).equals("A")) - assert(result(0).getString(1).equals("F")) - assert(result(0).getDouble(2) == 7578058.0) + assertResult(4)(result.length) + assertResult("A")(result(0).getString(0)) + assertResult("F")(result(0).getString(1)) + assertResult(7578058.0)(result(0).getDouble(2)) - assert(result(2).getString(0).equals("N")) - assert(result(2).getString(1).equals("O")) - assert(result(2).getDouble(2) == 7454519.0) + assertResult("N")(result(2).getString(0)) + assertResult("O")(result(2).getString(1)) + assertResult(7454519.0)(result(2).getDouble(2)) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) - assert(mergetreeScan.metrics("numFiles").value == 6) + assertResult(6)(mergetreeScan.metrics("numFiles").value) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert( + .mkString(",")) + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + .mkString(",")) + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_returnflag")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) - assert(addFiles.map(_.rows).sum == 750735) + assertResult(6)(addFiles.size) + assertResult(750735)(addFiles.map(_.rows).sum) } spark.sql("drop table lineitem_mergetree_partition_hdfs") } @@ -503,36 +500,35 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) - assert(!ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) + assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isDefined) if (sparkVersion.equals("3.2")) { assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).orderByKeyOption.isEmpty) } else { - assert( + assertResult("l_partkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_partkey")) + .mkString(",")) } assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_returnflag")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 12) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(12)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } spark.sql("drop table lineitem_mergetree_bucket_hdfs") } @@ -585,39 +581,38 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) - assert(!ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isDefined) + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_orderkey")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.nonEmpty) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_returnflag")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 12) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(12)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } val result = spark.read .format("clickhouse") .load(dataPath) .count() - assert(result == 600572) + assertResult(600572)(result) } } // scalastyle:off line.size.limit diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala index c5dc3a23754e..30f443265cae 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnS3Suite.scala @@ -55,6 +55,7 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite .set("spark.sql.autoBroadcastJoinThreshold", "10MB") .set("spark.sql.adaptive.enabled", "true") .set("spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level", "error") + // .set("spark.gluten.sql.columnar.backend.ch.runtime_config.path", "/data") // for local test } override protected def beforeEach(): Unit = { @@ -152,7 +153,7 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -164,8 +165,8 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 1) - assert(addFiles.head.rows == 600572) + assertResult(1)(addFiles.size) + assertResult(600572)(addFiles.head.rows) } spark.sql("drop table lineitem_mergetree_s3") // clean up } @@ -237,7 +238,7 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) @@ -245,24 +246,22 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_shipdate,l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_shipdate,l_orderkey")) - assert( + .mkString(",")) + assertResult("l_shipdate")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_shipdate")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.isEmpty) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 1) - assert(addFiles.head.rows == 600572) + assertResult(1)(addFiles.size) + assertResult(600572)(addFiles.head.rows) } spark.sql("drop table lineitem_mergetree_orderbykey_s3") } @@ -399,51 +398,49 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite runTPCHQueryBySQL(1, sqlStr, compareResult = false) { df => val result = df.collect() - assert(result.length == 4) - assert(result(0).getString(0).equals("A")) - assert(result(0).getString(1).equals("F")) - assert(result(0).getDouble(2) == 7578058.0) + assertResult(4)(result.length) + assertResult("A")(result(0).getString(0)) + assertResult("F")(result(0).getString(1)) + assertResult(7578058.0)(result(0).getDouble(2)) - assert(result(2).getString(0).equals("N")) - assert(result(2).getString(1).equals("O")) - assert(result(2).getDouble(2) == 7454519.0) + assertResult("N")(result(2).getString(0)) + assertResult("O")(result(2).getString(1)) + assertResult(7454519.0)(result(2).getDouble(2)) val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) - assert(mergetreeScan.metrics("numFiles").value == 6) + assertResult(6)(mergetreeScan.metrics("numFiles").value) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert( + .mkString(",")) + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .primaryKeyOption .get - .mkString(",") - .equals("l_orderkey")) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + .mkString(",")) + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_returnflag")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 6) - assert(addFiles.map(_.rows).sum == 750735) + assertResult(6)(addFiles.size) + assertResult(750735)(addFiles.map(_.rows).sum) } spark.sql("drop table lineitem_mergetree_partition_s3") @@ -517,36 +514,35 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) - assert(!ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) + assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isDefined) if (sparkVersion.equals("3.2")) { assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).orderByKeyOption.isEmpty) } else { - assert( + assertResult("l_partkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_partkey")) + .mkString(",")) } assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.isEmpty) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_returnflag")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 12) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(12)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } spark.sql("drop table lineitem_mergetree_bucket_s3") } @@ -599,39 +595,38 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val fileIndex = mergetreeScan.relation.location.asInstanceOf[TahoeFileIndex] assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).clickhouseTableConfigs.nonEmpty) - assert(!ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isEmpty) - assert( + assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).bucketOption.isDefined) + assertResult("l_orderkey")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) .orderByKeyOption .get - .mkString(",") - .equals("l_orderkey")) + .mkString(",")) assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).primaryKeyOption.nonEmpty) - assert(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size == 1) - assert( + assertResult(1)(ClickHouseTableV2.getTable(fileIndex.deltaLog).partitionColumns.size) + assertResult("l_returnflag")( ClickHouseTableV2 .getTable(fileIndex.deltaLog) - .partitionColumns(0) - .equals("l_returnflag")) + .partitionColumns + .head) val addFiles = fileIndex.matchingFiles(Nil, Nil).map(f => f.asInstanceOf[AddMergeTreeParts]) - assert(addFiles.size == 12) - assert(addFiles.map(_.rows).sum == 600572) + assertResult(12)(addFiles.size) + assertResult(600572)(addFiles.map(_.rows).sum) } val result = spark.read .format("clickhouse") .load(dataPath) .count() - assert(result == 600572) + assertResult(600572)(result) } test("test mergetree insert with optimize basic") { @@ -639,8 +634,8 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite val dataPath = s"s3a://$BUCKET_NAME/$tableName" withSQLConf( - ("spark.databricks.delta.optimize.minFileSize" -> "200000000"), - ("spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true") + "spark.databricks.delta.optimize.minFileSize" -> "200000000", + "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true" ) { spark.sql(s""" |DROP TABLE IF EXISTS $tableName; @@ -654,7 +649,7 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite |""".stripMargin) val ret = spark.sql(s"select count(*) from $tableName").collect() - assert(ret.apply(0).get(0) == 600572) + assertResult(600572)(ret.apply(0).get(0)) assert( !new File(s"$CH_DEFAULT_STORAGE_DIR/lineitem_mergetree_insert_optimize_basic").exists()) } @@ -713,22 +708,22 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite |""".stripMargin withSQLConf( - ("spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> "true")) { + "spark.gluten.sql.columnar.backend.ch.runtime_settings.enabled_driver_filter_mergetree_index" -> "true") { runTPCHQueryBySQL(6, sqlStr) { df => val scanExec = collect(df.queryExecution.executedPlan) { case f: FileSourceScanExecTransformer => f } - assert(scanExec.size == 1) + assertResult(1)(scanExec.size) - val mergetreeScan = scanExec(0) + val mergetreeScan = scanExec.head assert(mergetreeScan.nodeName.startsWith("Scan mergetree")) val plans = collect(df.queryExecution.executedPlan) { case scanExec: BasicScanExecTransformer => scanExec } - assert(plans.size == 1) - assert(plans(0).getSplitInfos.size == 1) + assertResult(1)(plans.size) + assertResult(1)(plans.head.getSplitInfos.size) } } } diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 2bbb2945334b..1e3ac8d88ea9 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240616 -CH_COMMIT=803ee50cdb9fd56a5d77c710da1cbd071a74d1da +CH_BRANCH=rebase_ch/20240620 +CH_COMMIT=f9c3886a767 diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp index 259af5698aa1..c1f2391a282c 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp @@ -87,8 +87,7 @@ SparkMergeTreeWriter::SparkMergeTreeWriter( metadata_snapshot = storage->getInMemoryMetadataPtr(); header = metadata_snapshot->getSampleBlock(); const DB::Settings & settings = context->getSettingsRef(); - squashing_transform - = std::make_unique(settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes); + squashing = std::make_unique(header, settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes); if (!partition_dir.empty()) extractPartitionValues(partition_dir, partition_values); @@ -105,25 +104,33 @@ SparkMergeTreeWriter::SparkMergeTreeWriter( merge_limit_parts = limit_cnt_field.get() <= 0 ? merge_limit_parts : limit_cnt_field.get(); } -void SparkMergeTreeWriter::write(DB::Block & block) +void SparkMergeTreeWriter::write(const DB::Block & block) { auto new_block = removeColumnSuffix(block); if (auto converter = ActionsDAG::makeConvertingActions( new_block.getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), DB::ActionsDAG::MatchColumnsMode::Position)) ExpressionActions(converter).execute(new_block); - if (auto add_block = squashing_transform->add(new_block)) + bool has_part = chunkToPart(squashing->add({new_block.getColumns(), new_block.rows()})); + + if (has_part && merge_after_insert) + checkAndMerge(); +} + +bool SparkMergeTreeWriter::chunkToPart(Chunk && chunk) +{ + if (chunk.hasChunkInfo()) { - bool has_part = blockToPart(add_block); - if (has_part && merge_after_insert) - checkAndMerge(); + Chunk squash_chunk = DB::Squashing::squash(std::move(chunk)); + Block result = header.cloneWithColumns(squash_chunk.getColumns()); + return blockToPart(result); } + return false; } bool SparkMergeTreeWriter::blockToPart(Block & block) { - auto blocks_with_partition - = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), 10, metadata_snapshot, context); + auto blocks_with_partition = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), 10, metadata_snapshot, context); if (blocks_with_partition.empty()) return false; @@ -180,12 +187,7 @@ void SparkMergeTreeWriter::manualFreeMemory(size_t before_write_memory) void SparkMergeTreeWriter::finalize() { - if (auto block = squashing_transform->add({})) - { - if (block.rows()) - blockToPart(block); - } - + chunkToPart(squashing->flush()); if (merge_after_insert) finalizeMerge(); diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h index 5c4b66403303..2b07521ede3a 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h @@ -17,7 +17,7 @@ #pragma once #include -#include +#include #include #include #include @@ -59,13 +59,15 @@ class SparkMergeTreeWriter const String & partition_dir_ = "", const String & bucket_dir_ = ""); - void write(DB::Block & block); + void write(const DB::Block & block); void finalize(); std::vector getAllPartInfo(); private: - void - writeTempPart(MergeTreeDataWriter::TemporaryPart & temp_part, DB::BlockWithPartition & block_with_partition, const DB::StorageMetadataPtr & metadata_snapshot); + void writeTempPart( + MergeTreeDataWriter::TemporaryPart & temp_part, + DB::BlockWithPartition & block_with_partition, + const DB::StorageMetadataPtr & metadata_snapshot); DB::MergeTreeDataWriter::TemporaryPart writeTempPartAndFinalize(DB::BlockWithPartition & block_with_partition, const DB::StorageMetadataPtr & metadata_snapshot); void checkAndMerge(bool force = false); @@ -75,6 +77,7 @@ class SparkMergeTreeWriter void saveMetadata(); void commitPartToRemoteStorageIfNeeded(); void finalizeMerge(); + bool chunkToPart(Chunk && chunk); bool blockToPart(Block & block); CustomStorageMergeTreePtr storage = nullptr; @@ -87,7 +90,7 @@ class SparkMergeTreeWriter String bucket_dir; DB::ContextPtr context; - std::unique_ptr squashing_transform; + std::unique_ptr squashing; int part_num = 1; ConcurrentDeque new_parts; std::unordered_map partition_values; From 35695d1235017670cc11c2cfae71528bab36c307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Thu, 20 Jun 2024 16:44:24 +0800 Subject: [PATCH 310/402] [VL][Core] Turn off InputFileNameReplaceRule by default --- .../backendsapi/velox/VeloxSparkPlanExecApi.scala | 13 ++++++++----- .../execution/ScalarFunctionsValidateSuite.scala | 10 +++++++--- .../main/scala/org/apache/gluten/GlutenConfig.scala | 11 +++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 71930d7e0f47..1f868c4c2044 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -805,11 +805,14 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { * * @return */ - override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = List( - BloomFilterMightContainJointRewriteRule.apply, - ArrowScanReplaceRule.apply, - InputFileNameReplaceRule.apply - ) + override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = { + val buf: ListBuffer[SparkSession => Rule[SparkPlan]] = + ListBuffer(BloomFilterMightContainJointRewriteRule.apply, ArrowScanReplaceRule.apply) + if (GlutenConfig.getConf.enableInputFileNameReplaceRule) { + buf += InputFileNameReplaceRule.apply + } + buf.result + } /** * Generate extended columnar pre-rules. diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index d08ba11ee787..11eaa3289cab 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -624,9 +624,13 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } test("Test input_file_name function") { - runQueryAndCompare("""SELECT input_file_name(), l_orderkey - | from lineitem limit 100""".stripMargin) { - checkGlutenOperatorMatch[ProjectExecTransformer] + withSQLConf( + "spark.gluten.sql.enableInputFileNameReplaceRule" -> "true" + ) { + runQueryAndCompare("""SELECT input_file_name(), l_orderkey + | from lineitem limit 100""".stripMargin) { + checkGlutenOperatorMatch[ProjectExecTransformer] + } } } diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 462032488548..148e8cdc067c 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -37,6 +37,7 @@ case class GlutenNumaBindingInfo( class GlutenConfig(conf: SQLConf) extends Logging { import GlutenConfig._ + def enableInputFileNameReplaceRule: Boolean = conf.getConf(INPUT_FILE_NAME_REPLACE_RULE_ENABLED) def enableAnsiMode: Boolean = conf.ansiEnabled def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED) @@ -750,6 +751,16 @@ object GlutenConfig { .booleanConf .createWithDefault(GLUTEN_ENABLE_BY_DEFAULT) + val INPUT_FILE_NAME_REPLACE_RULE_ENABLED = + buildConf("spark.gluten.sql.enableInputFileNameReplaceRule") + .internal() + .doc( + "Experimental: This config apply for velox backend to specify whether to enable " + + "inputFileNameReplaceRule to support offload input_file_name " + + "expression to native.") + .booleanConf + .createWithDefault(false) + // FIXME the option currently controls both JVM and native validation against a Substrait plan. val NATIVE_VALIDATION_ENABLED = buildConf("spark.gluten.sql.enable.native.validation") From 951ecb9f490165b84519a1721e404b3e35dfd968 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 20 Jun 2024 17:01:18 +0800 Subject: [PATCH 311/402] [CORE] Bump scalawarts version to prepare for Scala 2.13 support (#6154) --- pom.xml | 4 ++-- shims/common/pom.xml | 1 + shims/spark32/pom.xml | 1 + shims/spark33/pom.xml | 1 + shims/spark34/pom.xml | 1 + shims/spark35/pom.xml | 1 + .../src/main/scala/org/apache/gluten/integration/Suite.scala | 5 ++++- 7 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 87b60e40ea85..81ce0e5d462a 100644 --- a/pom.xml +++ b/pom.xml @@ -635,8 +635,8 @@ io.github.zhztheplayer.scalawarts - scalawarts - 0.1.1 + scalawarts_${scala.binary.version} + 0.1.2 ${scala.recompile.mode} diff --git a/shims/common/pom.xml b/shims/common/pom.xml index adb4112ff2c4..adf9da7c624e 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -61,6 +61,7 @@ -Wconf:cat=deprecation:silent + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass diff --git a/shims/spark32/pom.xml b/shims/spark32/pom.xml index 4ee8889cf9a9..2a3ed7400c9c 100644 --- a/shims/spark32/pom.xml +++ b/shims/spark32/pom.xml @@ -108,6 +108,7 @@ -Wconf:cat=deprecation:silent + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass diff --git a/shims/spark33/pom.xml b/shims/spark33/pom.xml index 23e35718a4af..725d1f8edf01 100644 --- a/shims/spark33/pom.xml +++ b/shims/spark33/pom.xml @@ -109,6 +109,7 @@ -Wconf:cat=deprecation:silent + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass diff --git a/shims/spark34/pom.xml b/shims/spark34/pom.xml index 42d4b830dfd0..9cc9bb409bea 100644 --- a/shims/spark34/pom.xml +++ b/shims/spark34/pom.xml @@ -109,6 +109,7 @@ -Wconf:cat=deprecation:silent + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass diff --git a/shims/spark35/pom.xml b/shims/spark35/pom.xml index f5658eee2a3f..27cd011ac2d6 100644 --- a/shims/spark35/pom.xml +++ b/shims/spark35/pom.xml @@ -109,6 +109,7 @@ -Wconf:cat=deprecation:silent + -P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala index 51e1777e25f8..070c43e9be2f 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala @@ -60,7 +60,10 @@ abstract class Suite( .setWarningOnOverriding("spark.storage.blockManagerSlaveTimeoutMs", "3600000") sessionSwitcher .defaultConf() - .setWarningOnOverriding("spark.executor.heartbeatInterval", "1s") // for keeping metrics updated + .setWarningOnOverriding("spark.executor.heartbeatInterval", "10s") + sessionSwitcher + .defaultConf() + .setWarningOnOverriding("spark.worker.timeout", "3600") sessionSwitcher .defaultConf() .setWarningOnOverriding("spark.executor.metrics.pollingInterval", "0") From b4ac7d9f10496c74a4bd215b6588733bfdec1715 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 20 Jun 2024 18:06:13 +0800 Subject: [PATCH 312/402] [VL] Daily Update Velox Version (2024_06_20) (#6158) --- cpp/velox/CMakeLists.txt | 3 +++ ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 4eed625628f3..c2d690a7e055 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -264,6 +264,9 @@ macro(ADD_VELOX_DEPENDENCIES) add_velox_dependency( dwio::dwrf::reader "${VELOX_COMPONENTS_PATH}/dwio/dwrf/reader/libvelox_dwio_dwrf_reader.a") + add_velox_dependency( + dwio::orc::reader + "${VELOX_COMPONENTS_PATH}/dwio/orc/reader/libvelox_dwio_orc_reader.a") add_velox_dependency( dwio::dwrf::utils "${VELOX_COMPONENTS_PATH}/dwio/dwrf/utils/libvelox_dwio_dwrf_utils.a") diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 26374c2be504..1b0a3ebed1ee 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_19 +VELOX_BRANCH=2024_06_20 VELOX_HOME="" #Set on run gluten on HDFS From 88394d17230092d09edf644b0e1820fd31732f9b Mon Sep 17 00:00:00 2001 From: zhouyifan279 <88070094+zhouyifan279@users.noreply.github.com> Date: Fri, 21 Jun 2024 03:12:34 +0800 Subject: [PATCH 313/402] [BUILD] Syntax error when run `./dev/builddeps-veloxbe.sh --enable_s3=ON` (#6169) Small fix of build script --- ep/build-velox/src/get_velox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 1b0a3ebed1ee..e4f71214efcf 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -108,6 +108,7 @@ function process_setup_ubuntu { if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh # it's used for velox CI + sed -i '/rpm -i minio-20220526054841.0.0.x86_64.rpm/a \ \ echo "Skip installing minio"' scripts/setup-adapters.sh sed -i 's/rpm -i minio-20220526054841.0.0.x86_64.rpm/#rpm -i minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh fi if [ $ENABLE_GCS == "ON" ]; then From 5cd40594454a2ce538340eadbf021943a79d9360 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 21 Jun 2024 13:06:59 +0800 Subject: [PATCH 314/402] [VL] Daily Update Velox Version (2024_06_21) (#6173) --- ep/build-velox/src/get_velox.sh | 2 +- ep/build-velox/src/modify_velox.patch | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index e4f71214efcf..a0a7baa0da45 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_20 +VELOX_BRANCH=2024_06_21 VELOX_HOME="" #Set on run gluten on HDFS diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch index 81560917d620..aee406c3eae0 100644 --- a/ep/build-velox/src/modify_velox.patch +++ b/ep/build-velox/src/modify_velox.patch @@ -68,14 +68,6 @@ index 5c7bf770a..9f897f577 100644 if(NOT TARGET gflags::gflags) # This is a bit convoluted, but we want to be able to use gflags::gflags as a # target even when velox is built as a subproject which uses -@@ -441,7 +446,7 @@ if(${VELOX_BUILD_MINIMAL_WITH_DWIO} - - # Locate or build protobuf. - set_source(Protobuf) -- resolve_dependency(Protobuf 3.21.4 EXACT) -+ resolve_dependency(Protobuf 3.21 EXACT) - include_directories(${Protobuf_INCLUDE_DIRS}) - endif() diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index ce4c24dbe..785a2acc6 100644 From 7a4a07ffc2967997af8826b920dda860641ef2b6 Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Fri, 21 Jun 2024 14:10:19 +0800 Subject: [PATCH 315/402] [CH] support Levenshtein distance (#6108) [CH] support Levenshtein distance --------- Co-authored-by: liuneng1994 --- .../scala/org/apache/gluten/expression/ExpressionMappings.scala | 1 + .../apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala | 1 - .../apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala | 1 - .../scala/org/apache/gluten/expression/ExpressionNames.scala | 1 + 4 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index f0082456fb18..678ba38172eb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -101,6 +101,7 @@ object ExpressionMappings { Sig[Encode](ENCODE), Sig[Uuid](UUID), Sig[BitLength](BIT_LENGTH), + Sig[Levenshtein](LEVENSHTEIN), Sig[UnBase64](UNBASE64), Sig[Base64](BASE64), diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 19c9b2cf478f..8572ef54d5c8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -902,7 +902,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") - .exclude("Levenshtein distance") .exclude("soundex unit test") .exclude("replace") .exclude("overlay for string") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index da71110de3b4..50e7929e4619 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -911,7 +911,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") - .exclude("Levenshtein distance") .exclude("soundex unit test") .exclude("replace") .exclude("overlay for string") diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 112fa677d2cd..2be3fad9d39d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -127,6 +127,7 @@ object ExpressionNames { final val ENCODE = "encode" final val UUID = "uuid" final val BIT_LENGTH = "bit_length" + final val LEVENSHTEIN = "levenshteinDistance" final val UNBASE64 = "unbase64" final val BASE64 = "base64" From 812e976059f89f68d2060fb6b195d4732cc374d4 Mon Sep 17 00:00:00 2001 From: Yuan Date: Fri, 21 Jun 2024 17:30:17 +0800 Subject: [PATCH 316/402] [VL] update build package script (#5969) This patch fixes the build package script by 1) adding the arrow jar 2) use the latest docker image Signed-off-by: Yuan Zhou --- .github/workflows/build_bundle_package.yml | 25 ++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_bundle_package.yml b/.github/workflows/build_bundle_package.yml index 01ddd6f43857..8ce659366770 100644 --- a/.github/workflows/build_bundle_package.yml +++ b/.github/workflows/build_bundle_package.yml @@ -38,7 +38,7 @@ on: jobs: build-native-lib: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_03_17 + container: apache/gluten:gluten-vcpkg-builder_2024_05_29 steps: - uses: actions/checkout@v2 - name: Build Gluten velox third party @@ -53,11 +53,17 @@ jobs: export NUM_THREADS=4 ./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=OFF \ --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF - - uses: actions/upload-artifact@v2 + - name: Upload native libs + uses: actions/upload-artifact@v2 with: path: ./cpp/build/releases/ name: velox-native-lib-${{github.sha}} retention-days: 1 + - name: Upload Artifact Arrow Jar + uses: actions/upload-artifact@v2 + with: + path: /root/.m2/repository/org/apache/arrow/ + name: velox-arrow-jar-centos-7-${{github.sha}} build-bundle-package-ubuntu: if: startsWith(github.event.inputs.os, 'ubuntu') @@ -71,6 +77,11 @@ jobs: with: name: velox-native-lib-${{github.sha}} path: ./cpp/build/releases + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | apt-get update && \ @@ -99,6 +110,11 @@ jobs: with: name: velox-native-lib-${{github.sha}} path: ./cpp/build/releases + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \ @@ -130,6 +146,11 @@ jobs: with: name: velox-native-lib-${{github.sha}} path: ./cpp/build/releases + - name: Download All Arrow Jar Artifacts + uses: actions/download-artifact@v2 + with: + name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true && \ From e25ab2e7adc197ada9285e18f4e35544fec4e3fe Mon Sep 17 00:00:00 2001 From: "shuai.xu" Date: Fri, 21 Jun 2024 18:10:16 +0800 Subject: [PATCH 317/402] [GLUTEN-4451] [CH] fix header maybe changed by FilterTransform (#6166) What changes were proposed in this pull request? Rollback header if changed in FilterTransform (Fixes: #4451) How was this patch tested? This patch was tested by integration tests. --- ...enClickHouseTPCHSaltNullParquetSuite.scala | 50 +++++++++++++++++++ .../local-engine/Parser/FilterRelParser.cpp | 7 ++- .../Parser/SerializedPlanParser.cpp | 13 +++++ .../Parser/SerializedPlanParser.h | 1 + 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 1d3bbec848bc..5040153320fc 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2638,5 +2638,55 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("drop table test_tbl_5910_0") spark.sql("drop table test_tbl_5910_1") } + + test("GLUTEN-4451: Fix schema may be changed by filter") { + val create_sql = + """ + |create table if not exists test_tbl_4451( + | month_day string, + | month_dif int, + | is_month_new string, + | country string, + | os string, + | mr bigint + |) using parquet + |PARTITIONED BY ( + | day string, + | app_name string) + |""".stripMargin + val insert_sql1 = + "INSERT into test_tbl_4451 partition (day='2024-06-01', app_name='abc') " + + "values('2024-06-01', 0, '1', 'CN', 'iOS', 100)" + val insert_sql2 = + "INSERT into test_tbl_4451 partition (day='2024-06-01', app_name='abc') " + + "values('2024-06-01', 0, '1', 'CN', 'iOS', 50)" + val insert_sql3 = + "INSERT into test_tbl_4451 partition (day='2024-06-01', app_name='abc') " + + "values('2024-06-01', 1, '1', 'CN', 'iOS', 80)" + spark.sql(create_sql) + spark.sql(insert_sql1) + spark.sql(insert_sql2) + spark.sql(insert_sql3) + val select_sql = + """ + |SELECT * FROM ( + | SELECT + | month_day, + | country, + | if(os = 'ALite','Android',os) AS os, + | is_month_new, + | nvl(sum(if(month_dif = 0, mr, 0)),0) AS `month0_n`, + | nvl(sum(if(month_dif = 1, mr, 0)) / sum(if(month_dif = 0, mr, 0)),0) AS `month1_rate`, + | '2024-06-18' as day, + | app_name + | FROM test_tbl_4451 + | GROUP BY month_day,country,if(os = 'ALite','Android',os),is_month_new,app_name + |) tt + |WHERE month0_n > 0 AND month1_rate <= 1 AND os IN ('all','Android','iOS') + | AND app_name IS NOT NULL + |""".stripMargin + compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) + spark.sql("drop table test_tbl_4451") + } } // scalastyle:on line.size.limit diff --git a/cpp-ch/local-engine/Parser/FilterRelParser.cpp b/cpp-ch/local-engine/Parser/FilterRelParser.cpp index 4c71cc3126af..e0098f747c2a 100644 --- a/cpp-ch/local-engine/Parser/FilterRelParser.cpp +++ b/cpp-ch/local-engine/Parser/FilterRelParser.cpp @@ -59,7 +59,12 @@ DB::QueryPlanPtr FilterRelParser::parse(DB::QueryPlanPtr query_plan, const subst filter_step->setStepDescription("WHERE"); steps.emplace_back(filter_step.get()); query_plan->addStep(std::move(filter_step)); - + + // header maybe changed, need to rollback it + if (!blocksHaveEqualStructure(input_header, query_plan->getCurrentDataStream().header)) { + steps.emplace_back(getPlanParser()->addRollbackFilterHeaderStep(query_plan, input_header)); + } + // remove nullable auto * remove_null_step = getPlanParser()->addRemoveNullableStep(*query_plan, non_nullable_columns); if (remove_null_step) diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 5f2c9cc33150..40e01e3052a3 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -335,6 +335,19 @@ IQueryPlanStep * SerializedPlanParser::addRemoveNullableStep(QueryPlan & plan, c return step_ptr; } +IQueryPlanStep * SerializedPlanParser::addRollbackFilterHeaderStep(QueryPlanPtr & query_plan, const Block & input_header) +{ + auto convert_actions_dag = ActionsDAG::makeConvertingActions( + query_plan->getCurrentDataStream().header.getColumnsWithTypeAndName(), + input_header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + auto expression_step = std::make_unique(query_plan->getCurrentDataStream(), convert_actions_dag); + expression_step->setStepDescription("Generator for rollback filter"); + auto * step_ptr = expression_step.get(); + query_plan->addStep(std::move(expression_step)); + return step_ptr; +} + DataTypePtr wrapNullableType(substrait::Type_Nullability nullable, DataTypePtr nested_type) { return wrapNullableType(nullable == substrait::Type_Nullability_NULLABILITY_NULLABLE, nested_type); diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index ccd5c0fdc4c8..45ff5a20b5ae 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -299,6 +299,7 @@ class SerializedPlanParser static std::string getFunctionName(const std::string & function_sig, const substrait::Expression_ScalarFunction & function); IQueryPlanStep * addRemoveNullableStep(QueryPlan & plan, const std::set & columns); + IQueryPlanStep * addRollbackFilterHeaderStep(QueryPlanPtr & query_plan, const Block & input_header); static ContextMutablePtr global_context; static Context::ConfigurationPtr config; From ef7b2c516bee6c82161a86574dbda91e05ab1b3a Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Fri, 21 Jun 2024 08:10:34 -0500 Subject: [PATCH 318/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240621) (#6170) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240621) * Fix Build/UT due to https://github.com/ClickHouse/ClickHouse/pull/65234 * fix style * refactor test --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 +- cpp-ch/local-engine/Parser/JoinRelParser.cpp | 2 +- .../Parser/MergeTreeRelParser.cpp | 2 +- .../local-engine/Parser/ProjectRelParser.cpp | 1 - .../Parser/SerializedPlanParser.cpp | 9 +- .../Parser/SerializedPlanParser.h | 10 +- cpp-ch/local-engine/local_engine_jni.cpp | 72 ++-- .../local-engine/tests/gluten_test_util.cpp | 4 +- cpp-ch/local-engine/tests/gtest_parser.cpp | 320 ++++++++++++++++++ 9 files changed, 373 insertions(+), 51 deletions(-) create mode 100644 cpp-ch/local-engine/tests/gtest_parser.cpp diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 1e3ac8d88ea9..4a3088e54309 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240620 -CH_COMMIT=f9c3886a767 +CH_BRANCH=rebase_ch/20240621 +CH_COMMIT=acf666c1c4f diff --git a/cpp-ch/local-engine/Parser/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/JoinRelParser.cpp index 937e449b0825..58b156c3cf6e 100644 --- a/cpp-ch/local-engine/Parser/JoinRelParser.cpp +++ b/cpp-ch/local-engine/Parser/JoinRelParser.cpp @@ -459,7 +459,7 @@ void JoinRelParser::addConvertStep(TableJoin & table_join, DB::QueryPlan & left, rename_dag->getOutputs()[pos] = &alias; } } - rename_dag->projectInput(); + QueryPlanStepPtr project_step = std::make_unique(right.getCurrentDataStream(), rename_dag); project_step->setStepDescription("Right Table Rename"); steps.emplace_back(project_step.get()); diff --git a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp index c36db6b7484a..b51b76b97415 100644 --- a/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp +++ b/cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp @@ -211,7 +211,7 @@ PrewhereInfoPtr MergeTreeRelParser::parsePreWhereInfo(const substrait::Expressio prewhere_info->prewhere_column_name = filter_name; prewhere_info->need_filter = true; prewhere_info->remove_prewhere_column = true; - prewhere_info->prewhere_actions->projectInput(false); + for (const auto & name : input.getNames()) prewhere_info->prewhere_actions->tryRestoreColumn(name); return prewhere_info; diff --git a/cpp-ch/local-engine/Parser/ProjectRelParser.cpp b/cpp-ch/local-engine/Parser/ProjectRelParser.cpp index caf779ac13bc..eb190101f170 100644 --- a/cpp-ch/local-engine/Parser/ProjectRelParser.cpp +++ b/cpp-ch/local-engine/Parser/ProjectRelParser.cpp @@ -99,7 +99,6 @@ ProjectRelParser::SplittedActionsDAGs ProjectRelParser::splitActionsDAGInGenerat std::unordered_set first_split_nodes(array_join_node->children.begin(), array_join_node->children.end()); auto first_split_result = actions_dag->split(first_split_nodes); res.before_array_join = first_split_result.first; - res.before_array_join->projectInput(true); array_join_node = findArrayJoinNode(first_split_result.second); std::unordered_set second_split_nodes = {array_join_node}; diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 40e01e3052a3..f9ea783a2bbd 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -234,6 +234,7 @@ std::shared_ptr SerializedPlanParser::expressionsToActionsDAG( throw Exception(ErrorCodes::BAD_ARGUMENTS, "unsupported projection type {}.", magic_enum::enum_name(expr.rex_type_case())); } actions_dag->project(required_columns); + actions_dag->appendInputsForUnusedColumns(header); return actions_dag; } @@ -1790,7 +1791,7 @@ QueryPlanPtr SerializedPlanParser::parse(const std::string & plan) QueryPlanPtr SerializedPlanParser::parseJson(const std::string & json_plan) { auto plan_ptr = std::make_unique(); - auto s = google::protobuf::util::JsonStringToMessage(absl::string_view(json_plan.c_str()), plan_ptr.get()); + auto s = google::protobuf::util::JsonStringToMessage(absl::string_view(json_plan), plan_ptr.get()); if (!s.ok()) throw Exception(ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from json string failed: {}", s.ToString()); return parse(std::move(plan_ptr)); @@ -1831,7 +1832,7 @@ void SerializedPlanParser::collectJoinKeys( } } -ActionsDAGPtr ASTParser::convertToActions(const NamesAndTypesList & name_and_types, const ASTPtr & ast) +ActionsDAG ASTParser::convertToActions(const NamesAndTypesList & name_and_types, const ASTPtr & ast) const { NamesAndTypesList aggregation_keys; ColumnNumbersList aggregation_keys_indexes_list; @@ -1840,9 +1841,9 @@ ActionsDAGPtr ASTParser::convertToActions(const NamesAndTypesList & name_and_typ ActionsMatcher::Data visitor_data( context, size_limits_for_set, - size_t(0), + static_cast(0), name_and_types, - std::make_shared(name_and_types), + ActionsDAG(name_and_types), std::make_shared(), false /* no_subqueries */, false /* no_makeset */, diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 45ff5a20b5ae..8964f42d9d02 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -112,7 +112,7 @@ static const std::map SCALAR_FUNCTIONS {"rand", "randCanonical"}, {"isnan", "isNaN"}, {"bin", "sparkBin"}, - {"rint", "sparkRint"}, + {"rint", "sparkRint"}, /// string functions {"like", "like"}, @@ -151,7 +151,7 @@ static const std::map SCALAR_FUNCTIONS {"initcap", "initcapUTF8"}, {"conv", "sparkConv"}, {"uuid", "generateUUIDv4"}, - {"levenshteinDistance", "editDistanceUTF8"}, + {"levenshteinDistance", "editDistanceUTF8"}, /// hash functions {"crc32", "CRC32"}, @@ -278,7 +278,7 @@ class SerializedPlanParser materialize_inputs.emplace_back(materialize_input); } - void addSplitInfo(std::string & split_info) { split_infos.emplace_back(std::move(split_info)); } + void addSplitInfo(std::string && split_info) { split_infos.emplace_back(std::move(split_info)); } int nextSplitInfoIndex() { @@ -419,6 +419,7 @@ class LocalExecutor : public BlockIterator RelMetricPtr getMetric() const { return metric; } void setMetric(RelMetricPtr metric_) { metric = metric_; } void setExtraPlanHolder(std::vector & extra_plan_holder_) { extra_plan_holder = std::move(extra_plan_holder_); } + private: std::unique_ptr writeBlockToSparkRow(DB::Block & block); @@ -434,7 +435,6 @@ class LocalExecutor : public BlockIterator DB::QueryPlanPtr current_query_plan; RelMetricPtr metric; std::vector extra_plan_holder; - }; @@ -450,7 +450,7 @@ class ASTParser ~ASTParser() = default; ASTPtr parseToAST(const Names & names, const substrait::Expression & rel); - ActionsDAGPtr convertToActions(const NamesAndTypesList & name_and_types, const ASTPtr & ast); + ActionsDAG convertToActions(const NamesAndTypesList & name_and_types, const ASTPtr & ast) const; private: ContextPtr context; diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 38f188293726..256f373c28b5 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -36,10 +36,14 @@ #include #include #include +#include +#include #include #include #include +#include #include +#include #include #include #include @@ -51,10 +55,6 @@ #include #include #include -#include -#include -#include -#include #ifdef __cplusplus @@ -269,13 +269,12 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ parser.addInputIter(iter, materialize_input); } - for (jsize i = 0, split_info_arr_size = env->GetArrayLength(split_infos); i < split_info_arr_size; i++) { + for (jsize i = 0, split_info_arr_size = env->GetArrayLength(split_infos); i < split_info_arr_size; i++) + { jbyteArray split_info = static_cast(env->GetObjectArrayElement(split_infos, i)); - jsize split_info_size = env->GetArrayLength(split_info); + std::string::size_type split_info_size = env->GetArrayLength(split_info); jbyte * split_info_addr = env->GetByteArrayElements(split_info, nullptr); - std::string split_info_str; - split_info_str.assign(reinterpret_cast(split_info_addr), split_info_size); - parser.addSplitInfo(split_info_str); + parser.addSplitInfo(std::string{reinterpret_cast(split_info_addr), split_info_size}); } jsize plan_size = env->GetArrayLength(plan); @@ -630,8 +629,7 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na .max_sort_buffer_size = static_cast(max_sort_buffer_size), .spill_firstly_before_stop = static_cast(spill_firstly_before_stop), .force_external_sort = static_cast(force_external_sort), - .force_mermory_sort = static_cast(force_memory_sort) - }; + .force_mermory_sort = static_cast(force_memory_sort)}; auto name = jstring2string(env, short_name); local_engine::SplitterHolder * splitter; if (prefer_spill) @@ -696,8 +694,7 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na .throw_if_memory_exceed = static_cast(throw_if_memory_exceed), .flush_block_buffer_before_evict = static_cast(flush_block_buffer_before_evict), .force_external_sort = static_cast(force_external_sort), - .force_mermory_sort = static_cast(force_memory_sort) - }; + .force_mermory_sort = static_cast(force_memory_sort)}; auto name = jstring2string(env, short_name); local_engine::SplitterHolder * splitter; splitter = new local_engine::SplitterHolder{.splitter = std::make_unique(name, options, pusher)}; @@ -768,8 +765,8 @@ JNIEXPORT void Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_clo } // CHBlockConverterJniWrapper -JNIEXPORT jobject -Java_org_apache_gluten_vectorized_CHBlockConverterJniWrapper_convertColumnarToRow(JNIEnv * env, jclass, jlong block_address, jintArray masks) +JNIEXPORT jobject Java_org_apache_gluten_vectorized_CHBlockConverterJniWrapper_convertColumnarToRow( + JNIEnv * env, jclass, jlong block_address, jintArray masks) { LOCAL_ENGINE_JNI_METHOD_START local_engine::CHColumnToSparkRow converter; @@ -958,21 +955,18 @@ JNIEXPORT jlong Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniW /// Parsing may fail when the number of recursive layers is large. /// Here, set a limit large enough to avoid this problem. /// Once this problem occurs, it is difficult to troubleshoot, because the pb of c++ will not provide any valid information - google::protobuf::io::CodedInputStream coded_in( - reinterpret_cast(plan_str.data()), static_cast(plan_str.size())); + google::protobuf::io::CodedInputStream coded_in(reinterpret_cast(plan_str.data()), static_cast(plan_str.size())); coded_in.SetRecursionLimit(100000); auto ok = plan_ptr->ParseFromCodedStream(&coded_in); if (!ok) throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from string failed"); - substrait::ReadRel::ExtensionTable extension_table = - local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); + substrait::ReadRel::ExtensionTable extension_table = local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); auto merge_tree_table = local_engine::MergeTreeRelParser::parseMergeTreeTable(extension_table); auto uuid = uuid_str + "_" + task_id; - auto * writer = new local_engine::SparkMergeTreeWriter( - merge_tree_table, query_context, uuid, partition_dir, bucket_dir); + auto * writer = new local_engine::SparkMergeTreeWriter(merge_tree_table, query_context, uuid, partition_dir, bucket_dir); env->ReleaseByteArrayElements(plan_, plan_buf_addr, JNI_ABORT); env->ReleaseByteArrayElements(split_info_, split_info_addr, JNI_ABORT); @@ -1044,8 +1038,8 @@ JNIEXPORT void Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWr LOCAL_ENGINE_JNI_METHOD_END(env, ) } -JNIEXPORT void -Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_writeToMergeTree(JNIEnv * env, jobject, jlong instanceId, jlong block_address) +JNIEXPORT void Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_writeToMergeTree( + JNIEnv * env, jobject, jlong instanceId, jlong block_address) { LOCAL_ENGINE_JNI_METHOD_START auto * writer = reinterpret_cast(instanceId); @@ -1054,7 +1048,8 @@ Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_writeToMe LOCAL_ENGINE_JNI_METHOD_END(env, ) } -JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_closeMergeTreeWriter(JNIEnv * env, jobject, jlong instanceId) +JNIEXPORT jstring +Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_closeMergeTreeWriter(JNIEnv * env, jobject, jlong instanceId) { LOCAL_ENGINE_JNI_METHOD_START auto * writer = reinterpret_cast(instanceId); @@ -1067,7 +1062,14 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn } JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_nativeMergeMTParts( - JNIEnv * env, jobject, jbyteArray plan_, jbyteArray split_info_, jstring uuid_, jstring task_id_, jstring partition_dir_, jstring bucket_dir_) + JNIEnv * env, + jobject, + jbyteArray plan_, + jbyteArray split_info_, + jstring uuid_, + jstring task_id_, + jstring partition_dir_, + jstring bucket_dir_) { LOCAL_ENGINE_JNI_METHOD_START @@ -1095,16 +1097,14 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn /// Parsing may fail when the number of recursive layers is large. /// Here, set a limit large enough to avoid this problem. /// Once this problem occurs, it is difficult to troubleshoot, because the pb of c++ will not provide any valid information - google::protobuf::io::CodedInputStream coded_in( - reinterpret_cast(plan_str.data()), static_cast(plan_str.size())); + google::protobuf::io::CodedInputStream coded_in(reinterpret_cast(plan_str.data()), static_cast(plan_str.size())); coded_in.SetRecursionLimit(100000); auto ok = plan_ptr->ParseFromCodedStream(&coded_in); if (!ok) throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from string failed"); - substrait::ReadRel::ExtensionTable extension_table = - local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); + substrait::ReadRel::ExtensionTable extension_table = local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); google::protobuf::StringValue table; table.ParseFromString(extension_table.detail().value()); auto merge_tree_table = local_engine::parseMergeTreeTableString(table.value()); @@ -1114,12 +1114,12 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn = local_engine::MergeTreeRelParser::copyToVirtualStorage(merge_tree_table, local_engine::SerializedPlanParser::global_context); local_engine::TempStorageFreer freer{temp_storage->getStorageID()}; // to release temp CustomStorageMergeTree with RAII - std::vector selected_parts - = local_engine::StorageMergeTreeFactory::instance().getDataPartsByNames(temp_storage->getStorageID(), "", merge_tree_table.getPartNames()); + std::vector selected_parts = local_engine::StorageMergeTreeFactory::instance().getDataPartsByNames( + temp_storage->getStorageID(), "", merge_tree_table.getPartNames()); std::unordered_map partition_values; - std::vector loaded = - local_engine::mergeParts(selected_parts, partition_values, uuid_str, temp_storage, partition_dir, bucket_dir); + std::vector loaded + = local_engine::mergeParts(selected_parts, partition_values, uuid_str, temp_storage, partition_dir, bucket_dir); std::vector res; for (auto & partPtr : loaded) @@ -1156,7 +1156,8 @@ JNIEXPORT jobject Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn partition_col_indice_vec.push_back(pIndice[i]); env->ReleaseIntArrayElements(partitionColIndice, pIndice, JNI_ABORT); - local_engine::BlockStripes bs = local_engine::BlockStripeSplitter::split(*block, partition_col_indice_vec, hasBucket, reserve_partition_columns); + local_engine::BlockStripes bs + = local_engine::BlockStripeSplitter::split(*block, partition_col_indice_vec, hasBucket, reserve_partition_columns); auto * addresses = env->NewLongArray(bs.block_addresses.size()); @@ -1366,7 +1367,8 @@ JNIEXPORT jlong Java_org_apache_gluten_memory_alloc_CHNativeMemoryAllocator_getD return -1; } -JNIEXPORT jlong Java_org_apache_gluten_memory_alloc_CHNativeMemoryAllocator_createListenableAllocator(JNIEnv * env, jclass, jobject listener) +JNIEXPORT jlong +Java_org_apache_gluten_memory_alloc_CHNativeMemoryAllocator_createListenableAllocator(JNIEnv * env, jclass, jobject listener) { LOCAL_ENGINE_JNI_METHOD_START auto listener_wrapper = std::make_shared(env->NewGlobalRef(listener)); diff --git a/cpp-ch/local-engine/tests/gluten_test_util.cpp b/cpp-ch/local-engine/tests/gluten_test_util.cpp index 7fdd32d1661b..0448092b960d 100644 --- a/cpp-ch/local-engine/tests/gluten_test_util.cpp +++ b/cpp-ch/local-engine/tests/gluten_test_util.cpp @@ -62,14 +62,14 @@ ActionsDAGPtr parseFilter(const std::string & filter, const AnotherRowType & nam size_limits_for_set, static_cast(0), name_and_types, - std::make_shared(name_and_types), + ActionsDAG(name_and_types), prepared_sets /* prepared_sets */, false /* no_subqueries */, false /* no_makeset */, false /* only_consts */, info); ActionsVisitor(visitor_data).visit(ast_exp); - return ActionsDAG::buildFilterActionsDAG({visitor_data.getActions()->getOutputs().back()}, node_name_to_input_column); + return ActionsDAG::buildFilterActionsDAG({visitor_data.getActions().getOutputs().back()}, node_name_to_input_column); } const char * get_data_dir() diff --git a/cpp-ch/local-engine/tests/gtest_parser.cpp b/cpp-ch/local-engine/tests/gtest_parser.cpp new file mode 100644 index 000000000000..cbe41c90c81a --- /dev/null +++ b/cpp-ch/local-engine/tests/gtest_parser.cpp @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +using namespace local_engine; +using namespace DB; + +std::string splitBinaryFromJson(const std::string & json) +{ + std::string binary; + substrait::ReadRel::LocalFiles local_files; + auto s = google::protobuf::util::JsonStringToMessage(absl::string_view(json), &local_files); + local_files.SerializeToString(&binary); + return binary; +} + +std::string JsonPlanFor65234() +{ + // Plan for https://github.com/ClickHouse/ClickHouse/pull/65234 + return R"( +{ + "extensions": [{ + "extensionFunction": { + "functionAnchor": 1, + "name": "is_not_null:str" + } + }, { + "extensionFunction": { + "functionAnchor": 2, + "name": "equal:str_str" + } + }, { + "extensionFunction": { + "functionAnchor": 3, + "name": "is_not_null:i64" + } + }, { + "extensionFunction": { + "name": "and:bool_bool" + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }] + }, + "columnTypes": ["NORMAL_COL", "NORMAL_COL"] + }, + "filter": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }, { + "value": { + "literal": { + "string": "EUROPE" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + } + } + } + }] + } + } + }] + } + }, + "advancedExtension": { + "optimization": { + "@type": "type.googleapis.com/google.protobuf.StringValue", + "value": "isMergeTree\u003d0\n" + } + } + } + }, + "condition": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }, { + "value": { + "literal": { + "string": "EUROPE" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + } + } + }] + } + }, + "names": ["r_regionkey#72"], + "outputSchema": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + } + }] +} +)"; +} + +TEST(SerializedPlanParser, PR65234) +{ + const std::string split + = R"({"items":[{"uriFile":"file:///part-00000-16caa751-9774-470c-bd37-5c84c53373c8-c000.snappy.parquet","length":"84633","parquet":{},"schema":{},"metadataColumns":[{}]}]}")"; + SerializedPlanParser parser(SerializedPlanParser::global_context); + parser.addSplitInfo(splitBinaryFromJson(split)); + parser.parseJson(JsonPlanFor65234()); +} From 1e06169cde0c6a22dc36d7c0af2a401bd73e1701 Mon Sep 17 00:00:00 2001 From: JiaKe Date: Sat, 22 Jun 2024 02:58:13 +0800 Subject: [PATCH 319/402] [GLUTEN-6151] Reset local property after finishing write operator (#6163) quick bug fix. Need to reset local property on fallback --- .../GlutenWriterColumnarRules.scala | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala index f9ad5201d8db..7063c3f67b80 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRules.scala @@ -162,19 +162,28 @@ object GlutenWriterColumnarRules { if write.getClass.getName == NOOP_WRITE && BackendsApiManager.getSettings.enableNativeWriteFiles() => injectFakeRowAdaptor(rc, rc.child) - case rc @ DataWritingCommandExec(cmd, child) - if BackendsApiManager.getSettings.supportNativeWrite(child.output.toStructType.fields) => - val format = getNativeFormat(cmd) - session.sparkContext.setLocalProperty( - "staticPartitionWriteOnly", - BackendsApiManager.getSettings.staticPartitionWriteOnly().toString) - // FIXME: We should only use context property if having no other approaches. - // Should see if there is another way to pass these options. - session.sparkContext.setLocalProperty("isNativeAppliable", format.isDefined.toString) - session.sparkContext.setLocalProperty("nativeFormat", format.getOrElse("")) - if (format.isDefined) { - injectFakeRowAdaptor(rc, child) + case rc @ DataWritingCommandExec(cmd, child) => + if (BackendsApiManager.getSettings.supportNativeWrite(child.output.toStructType.fields)) { + val format = getNativeFormat(cmd) + session.sparkContext.setLocalProperty( + "staticPartitionWriteOnly", + BackendsApiManager.getSettings.staticPartitionWriteOnly().toString) + // FIXME: We should only use context property if having no other approaches. + // Should see if there is another way to pass these options. + session.sparkContext.setLocalProperty("isNativeAppliable", format.isDefined.toString) + session.sparkContext.setLocalProperty("nativeFormat", format.getOrElse("")) + if (format.isDefined) { + injectFakeRowAdaptor(rc, child) + } else { + rc.withNewChildren(rc.children.map(apply)) + } } else { + session.sparkContext.setLocalProperty( + "staticPartitionWriteOnly", + BackendsApiManager.getSettings.staticPartitionWriteOnly().toString) + session.sparkContext.setLocalProperty("isNativeAppliable", "false") + session.sparkContext.setLocalProperty("nativeFormat", "") + rc.withNewChildren(rc.children.map(apply)) } case plan: SparkPlan => plan.withNewChildren(plan.children.map(apply)) From 4ed161be4e044322c7b3267d48dc6dffa40cae72 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 24 Jun 2024 08:54:15 +0800 Subject: [PATCH 320/402] [VL] RAS: Incorporate query plan's logical link into metadata model (#6165) --- .../columnar/enumerated/RemoveFilter.scala | 1 + .../planner/metadata/GlutenMetadata.scala | 36 ++--------- .../metadata/GlutenMetadataModel.scala | 21 +++--- .../gluten/planner/metadata/LogicalLink.scala | 53 +++++++++++++++ .../gluten/planner/metadata/Schema.scala | 64 +++++++++++++++++++ .../apache/gluten/planner/property/Conv.scala | 1 + .../ras/best/GroupBasedBestFinder.scala | 14 +++- .../apache/gluten/ras/OperationSuite.scala | 4 +- 8 files changed, 150 insertions(+), 44 deletions(-) create mode 100644 gluten-core/src/main/scala/org/apache/gluten/planner/metadata/LogicalLink.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/planner/metadata/Schema.scala diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index 5d7209dfbfb4..e2b8439fd218 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -42,6 +42,7 @@ object RemoveFilter extends RasRule[SparkPlan] { val filter = node.asInstanceOf[FilterExecTransformerBase] if (filter.isNoop()) { val out = NoopFilter(filter.child, filter.output) + out.copyTagsFrom(filter) return List(out) } List.empty diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala index e25f0a1f1c06..f66c5290e95f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala @@ -18,42 +18,18 @@ package org.apache.gluten.planner.metadata import org.apache.gluten.ras.Metadata -import org.apache.spark.sql.catalyst.expressions.Attribute - sealed trait GlutenMetadata extends Metadata { - import GlutenMetadata._ def schema(): Schema + def logicalLink(): LogicalLink } object GlutenMetadata { - def apply(schema: Schema): Metadata = { - Impl(schema) + def apply(schema: Schema, logicalLink: LogicalLink): Metadata = { + Impl(schema, logicalLink) } - private case class Impl(override val schema: Schema) extends GlutenMetadata - - case class Schema(output: Seq[Attribute]) { - private val hash = output.map(_.semanticHash()).hashCode() - - override def hashCode(): Int = { - hash - } - - override def equals(obj: Any): Boolean = obj match { - case other: Schema => - semanticEquals(other) - case _ => - false - } - - private def semanticEquals(other: Schema): Boolean = { - if (output.size != other.output.size) { - return false - } - output.zip(other.output).forall { - case (left, right) => - left.semanticEquals(right) - } - } + private case class Impl(override val schema: Schema, override val logicalLink: LogicalLink) + extends GlutenMetadata { + override def toString: String = s"$schema,$logicalLink" } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadataModel.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadataModel.scala index 6d1baa79db17..7b95f1383d04 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadataModel.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadataModel.scala @@ -16,7 +16,6 @@ */ package org.apache.gluten.planner.metadata -import org.apache.gluten.planner.metadata.GlutenMetadata.Schema import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Metadata, MetadataModel} @@ -31,18 +30,22 @@ object GlutenMetadataModel extends Logging { private object MetadataModelImpl extends MetadataModel[SparkPlan] { override def metadataOf(node: SparkPlan): Metadata = node match { case g: GroupLeafExec => throw new UnsupportedOperationException() - case other => GlutenMetadata(Schema(other.output)) + case other => + GlutenMetadata( + Schema(other.output), + other.logicalLink.map(LogicalLink(_)).getOrElse(LogicalLink.notFound)) } - override def dummy(): Metadata = GlutenMetadata(Schema(List())) + override def dummy(): Metadata = GlutenMetadata(Schema(List()), LogicalLink.notFound) override def verify(one: Metadata, other: Metadata): Unit = (one, other) match { - case (left: GlutenMetadata, right: GlutenMetadata) if left.schema() != right.schema() => - // We apply loose restriction on schema. Since Gluten still have some customized - // logics causing schema of an operator to change after being transformed. - // For example: https://github.com/apache/incubator-gluten/pull/5171 - logWarning(s"Warning: Schema mismatch: one: ${left.schema()}, other: ${right.schema()}") - case (left: GlutenMetadata, right: GlutenMetadata) if left == right => + case (left: GlutenMetadata, right: GlutenMetadata) => + implicitly[Verifier[Schema]].verify(left.schema(), right.schema()) + implicitly[Verifier[LogicalLink]].verify(left.logicalLink(), right.logicalLink()) case _ => throw new IllegalStateException(s"Metadata mismatch: one: $one, other $other") } } + + trait Verifier[T <: Any] { + def verify(one: T, other: T): Unit + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/LogicalLink.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/LogicalLink.scala new file mode 100644 index 000000000000..4c3bffd471ad --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/LogicalLink.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.planner.metadata + +import org.apache.gluten.planner.metadata.GlutenMetadataModel.Verifier + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Statistics} + +case class LogicalLink(plan: LogicalPlan) { + override def hashCode(): Int = System.identityHashCode(plan) + override def equals(obj: Any): Boolean = obj match { + // LogicalLink's comparison is based on ref equality of the logical plans being compared. + case LogicalLink(otherPlan) => plan eq otherPlan + case _ => false + } + + override def toString: String = s"${plan.nodeName}[${plan.stats.simpleString}]" +} + +object LogicalLink { + private case class LogicalLinkNotFound() extends logical.LeafNode { + override def output: Seq[Attribute] = List.empty + override def canEqual(that: Any): Boolean = throw new UnsupportedOperationException() + override def computeStats(): Statistics = Statistics(sizeInBytes = 0) + } + + val notFound = new LogicalLink(LogicalLinkNotFound()) + implicit val verifier: Verifier[LogicalLink] = new Verifier[LogicalLink] with Logging { + override def verify(one: LogicalLink, other: LogicalLink): Unit = { + // LogicalLink's comparison is based on ref equality of the logical plans being compared. + if (one != other) { + logWarning(s"Warning: Logical link mismatch: one: $one, other: $other") + } + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/Schema.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/Schema.scala new file mode 100644 index 000000000000..969d34d5cc82 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/Schema.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.planner.metadata + +import org.apache.gluten.planner.metadata.GlutenMetadataModel.Verifier + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Attribute + +case class Schema(output: Seq[Attribute]) { + private val hash = output.map(_.semanticHash()).hashCode() + + override def hashCode(): Int = { + hash + } + + override def equals(obj: Any): Boolean = obj match { + case other: Schema => + semanticEquals(other) + case _ => + false + } + + private def semanticEquals(other: Schema): Boolean = { + if (output.size != other.output.size) { + return false + } + output.zip(other.output).forall { + case (left, right) => + left.semanticEquals(right) + } + } + + override def toString: String = { + output.toString() + } +} + +object Schema { + implicit val verifier: Verifier[Schema] = new Verifier[Schema] with Logging { + override def verify(one: Schema, other: Schema): Unit = { + if (one != other) { + // We apply loose restriction on schema. Since Gluten still have some customized + // logics causing schema of an operator to change after being transformed. + // For example: https://github.com/apache/incubator-gluten/pull/5171 + logWarning(s"Warning: Schema mismatch: one: $one, other: $other") + } + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala index 475f6292094c..18db0f959491 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Conv.scala @@ -99,6 +99,7 @@ case class ConvEnforcerRule(reqConv: Conv) extends RasRule[SparkPlan] { } val transition = Conv.findTransition(conv, reqConv) val after = transition.apply(node) + after.copyTagsFrom(node) List(after) } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/best/GroupBasedBestFinder.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/best/GroupBasedBestFinder.scala index effebd41bb3b..1128ab8dec01 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/best/GroupBasedBestFinder.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/best/GroupBasedBestFinder.scala @@ -82,15 +82,23 @@ private object GroupBasedBestFinder { return Some(KnownCostPath(ras, path)) } val childrenGroups = can.getChildrenGroups(allGroups).map(gn => allGroups(gn.groupId())) - val maybeBestChildrenPaths: Seq[Option[RasPath[T]]] = childrenGroups.map { - childGroup => childrenGroupsOutput(childGroup).map(kcg => kcg.best().rasPath) + val maybeBestChildrenPaths: Seq[Option[KnownCostPath[T]]] = childrenGroups.map { + childGroup => childrenGroupsOutput(childGroup).map(kcg => kcg.best()) } if (maybeBestChildrenPaths.exists(_.isEmpty)) { // Node should only be solved when all children outputs exist. return None } val bestChildrenPaths = maybeBestChildrenPaths.map(_.get) - Some(KnownCostPath(ras, path.RasPath(ras, can, bestChildrenPaths).get)) + val kcp = KnownCostPath(ras, path.RasPath(ras, can, bestChildrenPaths.map(_.rasPath)).get) + // Cost should be in monotonically increasing basis. + bestChildrenPaths.map(_.cost).foreach { + childCost => + assert( + ras.costModel.costComparator().gteq(kcp.cost, childCost), + "Illegal decreasing cost") + } + Some(kcp) } override def solveGroup( diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala index 60ec2eedd410..e1ccfa1f44aa 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/OperationSuite.scala @@ -230,7 +230,7 @@ class OperationSuite extends AnyFunSuite { 48, Unary3(48, Unary3(48, Unary3(48, Unary3(48, Unary3(48, Unary3(48, Leaf(30)))))))))))) assert(costModel.costOfCount == 32) // TODO reduce this for performance - assert(costModel.costCompareCount == 20) // TODO reduce this for performance + assert(costModel.costCompareCount == 50) // TODO reduce this for performance } test("Cost evaluation count - max cost") { @@ -292,7 +292,7 @@ class OperationSuite extends AnyFunSuite { 48, Unary3(48, Unary3(48, Unary3(48, Unary3(48, Unary3(48, Unary3(48, Leaf(30)))))))))))) assert(costModel.costOfCount == 32) // TODO reduce this for performance - assert(costModel.costCompareCount == 20) // TODO reduce this for performance + assert(costModel.costCompareCount == 50) // TODO reduce this for performance } } From 9cceba6812d4de9883695f20bc506648083715f9 Mon Sep 17 00:00:00 2001 From: Xiduo You Date: Mon, 24 Jun 2024 09:07:09 +0800 Subject: [PATCH 321/402] [CORE] Fix the java.nio.file.NoSuchFileException: default in spark 3.5 (#6175) --- .../src/main/scala/org/apache/spark/HdfsConfGenerator.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala index 9756837d96e5..04272517e5bb 100644 --- a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala +++ b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala @@ -41,8 +41,8 @@ object HdfsConfGenerator extends Logging { addFileMethod.invoke(sc, path, Boolean.box(false), Boolean.box(true), Boolean.box(false)) // Overwrite the spark internal config `spark.app.initial.file.urls`, // so that the file can be available before initializing executor plugin. - assert(sc.addedFiles.nonEmpty) - sc.conf.set("spark.app.initial.file.urls", sc.addedFiles.keys.toSeq.mkString(",")) + assert(sc.listFiles.nonEmpty) + sc.conf.set("spark.app.initial.file.urls", sc.listFiles().mkString(",")) } private def ignoreKey(key: String): Boolean = { From eee234e398c9418b6f5f93dcfb142e0e0948711f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Mon, 24 Jun 2024 13:51:42 +0800 Subject: [PATCH 322/402] [GLUTEN-6122] Fix crash when driver send shutdown command to executor #6130 What changes were proposed in this pull request? Fix crash when driver send shutdown command to executor (Fixes: #6122) --- cpp-ch/local-engine/Common/CHUtil.cpp | 7 ++- .../Parser/SerializedPlanParser.cpp | 56 ++++++++++++++++++- .../Parser/SerializedPlanParser.h | 14 ++++- cpp-ch/local-engine/local_engine_jni.cpp | 11 +++- 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 937beae99a6b..be66d8ecc509 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -750,7 +750,7 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) size_t index_uncompressed_cache_size = config->getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); double index_uncompressed_cache_size_ratio = config->getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio); - + String index_mark_cache_policy = config->getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); size_t index_mark_cache_size = config->getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); double index_mark_cache_size_ratio = config->getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); @@ -919,7 +919,10 @@ void BackendInitializerUtil::updateConfig(const DB::ContextMutablePtr & context, void BackendFinalizerUtil::finalizeGlobally() { - // Make sure client caches release before ClientCacheRegistry + /// Make sure that all active LocalExecutor stop before spark executor shutdown, otherwise crash map happen. + LocalExecutor::cancelAll(); + + /// Make sure client caches release before ClientCacheRegistry ReadBufferBuilderFactory::instance().clean(); StorageMergeTreeFactory::clear(); auto & global_context = SerializedPlanParser::global_context; diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index f9ea783a2bbd..70db692c8009 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -2053,6 +2053,33 @@ void SerializedPlanParser::wrapNullable( SharedContextHolder SerializedPlanParser::shared_context; +std::unordered_map LocalExecutor::executors; +std::mutex LocalExecutor::executors_mutex; + +void LocalExecutor::cancelAll() +{ + std::lock_guard lock{executors_mutex}; + + for (auto & [handle, executor] : executors) + executor->asyncCancel(); + + for (auto & [handle, executor] : executors) + executor->waitCancelFinished(); +} + +void LocalExecutor::addExecutor(LocalExecutor * executor) +{ + std::lock_guard lock{executors_mutex}; + Int64 handle = reinterpret_cast(executor); + executors.emplace(handle, executor); +} + +void LocalExecutor::removeExecutor(Int64 handle) +{ + std::lock_guard lock{executors_mutex}; + executors.erase(handle); +} + LocalExecutor::~LocalExecutor() { if (context->getConfigRef().getBool("dump_pipeline", false)) @@ -2183,8 +2210,35 @@ Block * LocalExecutor::nextColumnar() void LocalExecutor::cancel() { - if (executor) + asyncCancel(); + waitCancelFinished(); +} + +void LocalExecutor::asyncCancel() +{ + if (executor && !is_cancelled) + { + LOG_INFO(&Poco::Logger::get("LocalExecutor"), "Cancel LocalExecutor {}", reinterpret_cast(this)); executor->cancel(); + } +} + +void LocalExecutor::waitCancelFinished() +{ + if (executor && !is_cancelled) + { + Stopwatch watch; + Chunk chunk; + while (executor->pull(chunk)) + ; + is_cancelled = true; + + LOG_INFO( + &Poco::Logger::get("LocalExecutor"), + "Finish cancel LocalExecutor {}, takes {} ms", + reinterpret_cast(this), + watch.elapsedMilliseconds()); + } } Block & LocalExecutor::getHeader() diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 8964f42d9d02..71cdca58a6ce 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -412,7 +412,7 @@ class LocalExecutor : public BlockIterator Block * nextColumnar(); bool hasNext(); - /// Stop execution, used when task receives shutdown command or executor receives SIGTERM signal + /// Stop execution and wait for pipeline exit, used when task receives shutdown command or executor receives SIGTERM signal void cancel(); Block & getHeader(); @@ -420,9 +420,16 @@ class LocalExecutor : public BlockIterator void setMetric(RelMetricPtr metric_) { metric = metric_; } void setExtraPlanHolder(std::vector & extra_plan_holder_) { extra_plan_holder = std::move(extra_plan_holder_); } + static void cancelAll(); + static void addExecutor(LocalExecutor * executor); + static void removeExecutor(Int64 handle); + private: std::unique_ptr writeBlockToSparkRow(DB::Block & block); + void asyncCancel(); + void waitCancelFinished(); + /// Dump processor runtime information to log std::string dumpPipeline(); @@ -435,6 +442,11 @@ class LocalExecutor : public BlockIterator DB::QueryPlanPtr current_query_plan; RelMetricPtr metric; std::vector extra_plan_holder; + std::atomic is_cancelled{false}; + + /// Record all active LocalExecutor in current executor to cancel them when executor receives shutdown command from driver. + static std::unordered_map executors; + static std::mutex executors_mutex; }; diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 256f373c28b5..bbc467879182 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -283,7 +283,8 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ plan_string.assign(reinterpret_cast(plan_address), plan_size); auto query_plan = parser.parse(plan_string); local_engine::LocalExecutor * executor = new local_engine::LocalExecutor(query_context); - LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); + local_engine::LocalExecutor::addExecutor(executor); + LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); executor->setMetric(parser.getMetric()); executor->setExtraPlanHolder(parser.extra_plan_holder); executor->execute(std::move(query_plan)); @@ -314,17 +315,19 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_BatchIterator_nativeCHNext(JNI JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeCancel(JNIEnv * env, jobject /*obj*/, jlong executor_address) { LOCAL_ENGINE_JNI_METHOD_START + local_engine::LocalExecutor::removeExecutor(executor_address); local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); executor->cancel(); - LOG_INFO(&Poco::Logger::get("jni"), "Cancel LocalExecutor {}", reinterpret_cast(executor)); + LOG_INFO(&Poco::Logger::get("jni"), "Cancel LocalExecutor {}", reinterpret_cast(executor)); LOCAL_ENGINE_JNI_METHOD_END(env, ) } JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeClose(JNIEnv * env, jobject /*obj*/, jlong executor_address) { LOCAL_ENGINE_JNI_METHOD_START + local_engine::LocalExecutor::removeExecutor(executor_address); local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); - LOG_INFO(&Poco::Logger::get("jni"), "Finalize LocalExecutor {}", reinterpret_cast(executor)); + LOG_INFO(&Poco::Logger::get("jni"), "Finalize LocalExecutor {}", reinterpret_cast(executor)); delete executor; LOCAL_ENGINE_JNI_METHOD_END(env, ) } @@ -1332,6 +1335,7 @@ Java_org_apache_gluten_vectorized_SimpleExpressionEval_createNativeInstance(JNIE plan_string.assign(reinterpret_cast(plan_address), plan_size); auto query_plan = parser.parse(plan_string); local_engine::LocalExecutor * executor = new local_engine::LocalExecutor(context); + local_engine::LocalExecutor::addExecutor(executor); executor->execute(std::move(query_plan)); env->ReleaseByteArrayElements(plan, plan_address, JNI_ABORT); return reinterpret_cast(executor); @@ -1341,6 +1345,7 @@ Java_org_apache_gluten_vectorized_SimpleExpressionEval_createNativeInstance(JNIE JNIEXPORT void Java_org_apache_gluten_vectorized_SimpleExpressionEval_nativeClose(JNIEnv * env, jclass, jlong instance) { LOCAL_ENGINE_JNI_METHOD_START + local_engine::LocalExecutor::removeExecutor(instance); local_engine::LocalExecutor * executor = reinterpret_cast(instance); delete executor; LOCAL_ENGINE_JNI_METHOD_END(env, ) From e0fcfe586efc7efb3ec0c349d5ca8b2371d969d4 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Mon, 24 Jun 2024 13:55:39 +0800 Subject: [PATCH 323/402] [GLUTEN-6178][CH] Add config to insert remote file system directly #6192 What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #6178) How was this patch tested? Test by ut --- ...nClickHouseMergeTreeWriteOnHDFSSuite.scala | 44 ++++++++++++++++++- cpp-ch/local-engine/Common/CHUtil.cpp | 3 +- cpp-ch/local-engine/Common/CHUtil.h | 4 +- .../Disks/ObjectStorages/GlutenDiskHDFS.cpp | 10 ++++- .../Disks/ObjectStorages/GlutenDiskHDFS.h | 2 + .../Mergetree/SparkMergeTreeWriter.cpp | 40 +++++++++-------- .../Storages/Mergetree/SparkMergeTreeWriter.h | 2 + 7 files changed, 83 insertions(+), 22 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala index 572d0cd50a6e..99b212059966 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala @@ -25,10 +25,12 @@ import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMerg import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.FileSystem +import org.apache.hadoop.fs.{FileSystem, Path} import java.io.File +import scala.concurrent.duration.DurationInt + // Some sqls' line length exceeds 100 // scalastyle:off line.size.limit @@ -614,5 +616,45 @@ class GlutenClickHouseMergeTreeWriteOnHDFSSuite .count() assertResult(600572)(result) } + + test("test mergetree insert with optimize basic") { + val tableName = "lineitem_mergetree_insert_optimize_basic_hdfs" + val dataPath = s"$HDFS_URL/test/$tableName" + + withSQLConf( + "spark.databricks.delta.optimize.minFileSize" -> "200000000", + "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert" -> "true", + "spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.insert_without_local_storage" -> "true", + "spark.gluten.sql.columnar.backend.ch.runtime_settings.min_insert_block_size_rows" -> "10000" + ) { + spark.sql(s""" + |DROP TABLE IF EXISTS $tableName; + |""".stripMargin) + + spark.sql(s""" + |CREATE TABLE IF NOT EXISTS $tableName + |USING clickhouse + |LOCATION '$dataPath' + |TBLPROPERTIES (storage_policy='__hdfs_main') + | as select * from lineitem + |""".stripMargin) + + val ret = spark.sql(s"select count(*) from $tableName").collect() + assertResult(600572)(ret.apply(0).get(0)) + val conf = new Configuration + conf.set("fs.defaultFS", HDFS_URL) + val fs = FileSystem.get(conf) + + eventually(timeout(60.seconds), interval(2.seconds)) { + val it = fs.listFiles(new Path(dataPath), true) + var files = 0 + while (it.hasNext) { + it.next() + files += 1 + } + assertResult(72)(files) + } + } + } } // scalastyle:off line.size.limit diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index be66d8ecc509..94cd38003bad 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -623,7 +623,8 @@ void BackendInitializerUtil::initSettings(std::map & b { /// Initialize default setting. settings.set("date_time_input_format", "best_effort"); - settings.set("mergetree.merge_after_insert", true); + settings.set(MERGETREE_MERGE_AFTER_INSERT, true); + settings.set(MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE, false); for (const auto & [key, value] : backend_conf_map) { diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 50de9461f4de..94e0f0168e11 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -35,7 +35,9 @@ class QueryPlan; namespace local_engine { -static const std::unordered_set BOOL_VALUE_SETTINGS{"mergetree.merge_after_insert"}; +static const String MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE = "mergetree.insert_without_local_storage"; +static const String MERGETREE_MERGE_AFTER_INSERT = "mergetree.merge_after_insert"; +static const std::unordered_set BOOL_VALUE_SETTINGS{MERGETREE_MERGE_AFTER_INSERT, MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE}; static const std::unordered_set LONG_VALUE_SETTINGS{ "optimize.maxfilesize", "optimize.minFileSize", "mergetree.max_num_part_per_merge_task"}; diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp index 07a7aa6bd006..f207ad232b4f 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.cpp @@ -52,7 +52,15 @@ void GlutenDiskHDFS::createDirectories(const String & path) void GlutenDiskHDFS::removeDirectory(const String & path) { DiskObjectStorage::removeDirectory(path); - hdfsDelete(hdfs_object_storage->getHDFSFS(), path.c_str(), 1); + String abs_path = "/" + path; + hdfsDelete(hdfs_object_storage->getHDFSFS(), abs_path.c_str(), 1); +} + +void GlutenDiskHDFS::removeRecursive(const String & path) +{ + DiskObjectStorage::removeRecursive(path); + String abs_path = "/" + path; + hdfsDelete(hdfs_object_storage->getHDFSFS(), abs_path.c_str(), 1); } DiskObjectStoragePtr GlutenDiskHDFS::createDiskObjectStorage() diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h index 222b9f8928a3..97a99f1deaba 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenDiskHDFS.h @@ -57,6 +57,8 @@ class GlutenDiskHDFS : public DB::DiskObjectStorage void removeDirectory(const String & path) override; + void removeRecursive(const String & path) override; + DB::DiskObjectStoragePtr createDiskObjectStorage() override; std::unique_ptr writeFile(const String& path, size_t buf_size, DB::WriteMode mode, diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp index c1f2391a282c..406f2aaa23df 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp @@ -69,11 +69,23 @@ SparkMergeTreeWriter::SparkMergeTreeWriter( , bucket_dir(bucket_dir_) , thread_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1, 1, 100000) { + const DB::Settings & settings = context->getSettingsRef(); + merge_after_insert = settings.get(MERGETREE_MERGE_AFTER_INSERT).get(); + insert_without_local_storage = settings.get(MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE).get(); + + Field limit_size_field; + if (settings.tryGet("optimize.minFileSize", limit_size_field)) + merge_min_size = limit_size_field.get() <= 0 ? merge_min_size : limit_size_field.get(); + + Field limit_cnt_field; + if (settings.tryGet("mergetree.max_num_part_per_merge_task", limit_cnt_field)) + merge_limit_parts = limit_cnt_field.get() <= 0 ? merge_limit_parts : limit_cnt_field.get(); + dest_storage = MergeTreeRelParser::parseStorage(merge_tree_table, SerializedPlanParser::global_context); + isRemoteStorage = dest_storage->getStoragePolicy()->getAnyDisk()->isRemote(); - if (dest_storage->getStoragePolicy()->getAnyDisk()->isRemote()) + if (useLocalStorage()) { - isRemoteStorage = true; temp_storage = MergeTreeRelParser::copyToDefaultPolicyStorage(merge_tree_table, SerializedPlanParser::global_context); storage = temp_storage; LOG_DEBUG( @@ -86,22 +98,14 @@ SparkMergeTreeWriter::SparkMergeTreeWriter( metadata_snapshot = storage->getInMemoryMetadataPtr(); header = metadata_snapshot->getSampleBlock(); - const DB::Settings & settings = context->getSettingsRef(); squashing = std::make_unique(header, settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes); if (!partition_dir.empty()) extractPartitionValues(partition_dir, partition_values); +} - Field is_merge; - if (settings.tryGet("mergetree.merge_after_insert", is_merge)) - merge_after_insert = is_merge.get(); - - Field limit_size_field; - if (settings.tryGet("optimize.minFileSize", limit_size_field)) - merge_min_size = limit_size_field.get() <= 0 ? merge_min_size : limit_size_field.get(); - - Field limit_cnt_field; - if (settings.tryGet("mergetree.max_num_part_per_merge_task", limit_cnt_field)) - merge_limit_parts = limit_cnt_field.get() <= 0 ? merge_limit_parts : limit_cnt_field.get(); +bool SparkMergeTreeWriter::useLocalStorage() const +{ + return !insert_without_local_storage && isRemoteStorage; } void SparkMergeTreeWriter::write(const DB::Block & block) @@ -161,7 +165,7 @@ void SparkMergeTreeWriter::manualFreeMemory(size_t before_write_memory) // it may alloc memory in current thread, and free on global thread. // Now, wo have not idea to clear global memory by used spark thread tracker. // So we manually correct the memory usage. - if (!isRemoteStorage) + if (isRemoteStorage && insert_without_local_storage) return; auto disk = storage->getStoragePolicy()->getAnyDisk(); @@ -219,7 +223,7 @@ void SparkMergeTreeWriter::saveMetadata() void SparkMergeTreeWriter::commitPartToRemoteStorageIfNeeded() { - if (!isRemoteStorage) + if (!useLocalStorage()) return; LOG_DEBUG( @@ -289,8 +293,8 @@ void SparkMergeTreeWriter::finalizeMerge() { for (const auto & disk : storage->getDisks()) { - auto full_path = storage->getFullPathOnDisk(disk); - disk->removeRecursive(full_path + "/" + tmp_part); + auto rel_path = storage->getRelativeDataPath() + "/" + tmp_part; + disk->removeRecursive(rel_path); } }); } diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h index 2b07521ede3a..13ac22394477 100644 --- a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h +++ b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h @@ -79,6 +79,7 @@ class SparkMergeTreeWriter void finalizeMerge(); bool chunkToPart(Chunk && chunk); bool blockToPart(Block & block); + bool useLocalStorage() const; CustomStorageMergeTreePtr storage = nullptr; CustomStorageMergeTreePtr dest_storage = nullptr; @@ -97,6 +98,7 @@ class SparkMergeTreeWriter std::unordered_set tmp_parts; DB::Block header; bool merge_after_insert; + bool insert_without_local_storage; FreeThreadPool thread_pool; size_t merge_min_size = 1024 * 1024 * 1024; size_t merge_limit_parts = 10; From 0ef2f8216b03f5f279d80c71baac30dbdb94199f Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:35:14 +0800 Subject: [PATCH 324/402] [VL] Support KnownNullable for Spark 3.5 (#6193) [VL] Support KnownNullable for Spark 3.5. --- .../execution/ScalarFunctionsValidateSuite.scala | 14 +++++++++++++- .../gluten/sql/shims/spark35/Spark35Shims.scala | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 11eaa3289cab..75b60addfa13 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -16,6 +16,7 @@ */ package org.apache.gluten.execution +import org.apache.spark.sql.execution.ProjectExec import org.apache.spark.sql.types._ import java.sql.Timestamp @@ -1145,7 +1146,18 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { runQueryAndCompare( "SELECT a, window.start, window.end, count(*) as cnt FROM" + " string_timestamp GROUP by a, window(b, '5 minutes') ORDER BY a, start;") { - checkGlutenOperatorMatch[ProjectExecTransformer] + df => + val executedPlan = getExecutedPlan(df) + assert( + executedPlan.exists(plan => plan.isInstanceOf[ProjectExecTransformer]), + s"Expect ProjectExecTransformer exists " + + s"in executedPlan:\n ${executedPlan.last}" + ) + assert( + !executedPlan.exists(plan => plan.isInstanceOf[ProjectExec]), + s"Expect ProjectExec doesn't exist " + + s"in executedPlan:\n ${executedPlan.last}" + ) } } } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 95571f166ebe..f6feae01a8b2 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -76,7 +76,9 @@ class Spark35Shims extends SparkShims { Sig[SplitPart](ExpressionNames.SPLIT_PART), Sig[Sec](ExpressionNames.SEC), Sig[Csc](ExpressionNames.CSC), - Sig[Empty2Null](ExpressionNames.EMPTY2NULL)) + Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE), + Sig[Empty2Null](ExpressionNames.EMPTY2NULL) + ) } override def aggregateExpressionMappings: Seq[Sig] = { From f07e348f4dfa5cf72a15a6986fd9524873072cdc Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:39:38 +0800 Subject: [PATCH 325/402] [VL] Daily Update Velox Version (2024_06_24) (#6187) f45966f17 by Deepak Majeti, Use separate headers for DWRF Reader Writer registration API (10132) 00485536f by Zac Wen, Switch to storage read if SSD cache load fails (10256) 3c2cc4b26 by Bikramjeet Vig, Fix NaN handling for in-predicate (10115) 18c4d5e2b by Kevin Wilfong, Capture MemoryArbitrationContext and ThreadDebugInfo in AsyncSource and restore them when invoking make (10186) 171174833 by Jimmy Lu, Count IO execution time in ExponentialBackoff retry policy (10286) 24f5aed63 by zhli1142015, Add support for DECIMAL input to greatest and least Spark functions (10195) 8faac7bf2 by zhli1142015, Add log Spark function (10243) c97e7fcc8 by Kevin Wilfong, Fix parallel spills lead to crashes in approx_percentile (10268) 54b2ce9a5 by Reetika Agrawal, Add benchmark for IcebergSplitReader (9849) dcd49ca38 by Krishna Pai, Restrict CAST of string to boolean (9833) ca5e409aa by xiaoxmeng, Only load stripe footer in buffer input support sync load (10276) 652cf372e by Zac Wen, Fix memory cache hit underreporting in ioStats (10272) a2366523d by yanngyoung, Add order by plan for memory arbitration fuzzer (10255) a5b443a70 by Wei He, Update header guards in files in velox/external/date to avoid collision (10269) --- cpp/velox/memory/VeloxMemoryManager.cc | 2 +- ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 60c79ffe8725..733eb4c4bc39 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -74,7 +74,7 @@ class ListenableArbitrator : public velox::memory::MemoryArbitrator { uint64_t targetBytes, bool allowSpill, bool allowAbort) override { - velox::memory::ScopedMemoryArbitrationContext ctx(nullptr); + velox::memory::ScopedMemoryArbitrationContext ctx((const velox::memory::MemoryPool*)nullptr); facebook::velox::exec::MemoryReclaimer::Stats status; VELOX_CHECK_EQ(pools.size(), 1, "Gluten only has one root pool"); std::lock_guard l(mutex_); // FIXME: Do we have recursive locking for this mutex? diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index a0a7baa0da45..d3ecddbdfa9a 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_21 +VELOX_BRANCH=2024_06_24 VELOX_HOME="" #Set on run gluten on HDFS From 1fbdbc41779321db3380bce0807b73389af64e1a Mon Sep 17 00:00:00 2001 From: Chang chen Date: Tue, 25 Jun 2024 07:13:39 +0800 Subject: [PATCH 326/402] [GLUTEN-6067][CH] [Part 2] Support CH backend with Spark3.5 - Prepare for supporting sink transform (#6197) [CH] [Part 2] Support CH backend with Spark3.5 - Prepare for supporting sink transform * [Refactor] remove duplicate codes * Add NativeWriteChecker * [Prepare to commit] getExtendedColumnarPostRules from Spark shim --- .../clickhouse/CHIteratorApi.scala | 143 ++-- .../clickhouse/CHSparkPlanExecApi.scala | 9 - .../execution/CHHashJoinExecTransformer.scala | 3 +- ...lutenClickHouseNativeWriteTableSuite.scala | 612 ++++++++---------- .../GlutenClickHouseTPCHMetricsSuite.scala | 2 +- .../spark/gluten/NativeWriteChecker.scala | 52 ++ .../velox/VeloxSparkPlanExecApi.scala | 9 - cpp-ch/local-engine/Common/CHUtil.cpp | 17 +- cpp-ch/local-engine/Common/CHUtil.h | 12 +- .../Parser/CHColumnToSparkRow.cpp | 2 +- .../Parser/SerializedPlanParser.cpp | 310 ++++----- .../Parser/SerializedPlanParser.h | 39 +- cpp-ch/local-engine/local_engine_jni.cpp | 39 +- .../tests/benchmark_local_engine.cpp | 80 +-- cpp-ch/local-engine/tests/gluten_test_util.h | 18 + .../local-engine/tests/gtest_local_engine.cpp | 22 +- cpp-ch/local-engine/tests/gtest_parser.cpp | 407 ++++-------- .../tests/json/clickhouse_pr_65234.json | 273 ++++++++ .../tests/json/gtest_local_engine_config.json | 269 ++++++++ .../json/read_student_option_schema.csv.json | 77 +++ .../gluten/backendsapi/SparkPlanExecApi.scala | 4 +- .../utils/SubstraitPlanPrinterUtil.scala | 35 +- 22 files changed, 1379 insertions(+), 1055 deletions(-) create mode 100644 backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala create mode 100644 cpp-ch/local-engine/tests/json/clickhouse_pr_65234.json create mode 100644 cpp-ch/local-engine/tests/json/gtest_local_engine_config.json create mode 100644 cpp-ch/local-engine/tests/json/read_student_option_schema.csv.json diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 941237629569..376e46ebe975 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.{GlutenConfig, GlutenNumaBindingInfo} +import org.apache.gluten.GlutenNumaBindingInfo import org.apache.gluten.backendsapi.IteratorApi import org.apache.gluten.execution._ import org.apache.gluten.expression.ConverterUtils @@ -61,6 +61,52 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { StructType(dataSchema) } + private def createNativeIterator( + splitInfoByteArray: Array[Array[Byte]], + wsPlan: Array[Byte], + materializeInput: Boolean, + inputIterators: Seq[Iterator[ColumnarBatch]]): BatchIterator = { + + /** Generate closeable ColumnBatch iterator. */ + val listIterator = + inputIterators + .map { + case i: CloseableCHColumnBatchIterator => i + case it => new CloseableCHColumnBatchIterator(it) + } + .map(it => new ColumnarNativeIterator(it.asJava).asInstanceOf[GeneralInIterator]) + .asJava + new CHNativeExpressionEvaluator().createKernelWithBatchIterator( + wsPlan, + splitInfoByteArray, + listIterator, + materializeInput + ) + } + + private def createCloseIterator( + context: TaskContext, + pipelineTime: SQLMetric, + updateNativeMetrics: IMetrics => Unit, + updateInputMetrics: Option[InputMetricsWrapper => Unit] = None, + nativeIter: BatchIterator): CloseableCHColumnBatchIterator = { + + val iter = new CollectMetricIterator( + nativeIter, + updateNativeMetrics, + updateInputMetrics, + updateInputMetrics.map(_ => context.taskMetrics().inputMetrics).orNull) + + context.addTaskFailureListener( + (ctx, _) => { + if (ctx.isInterrupted()) { + iter.cancel() + } + }) + context.addTaskCompletionListener[Unit](_ => iter.close()) + new CloseableCHColumnBatchIterator(iter, Some(pipelineTime)) + } + // only set file schema for text format table private def setFileSchemaForLocalFiles( localFilesNode: LocalFilesNode, @@ -198,45 +244,24 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { inputIterators: Seq[Iterator[ColumnarBatch]] = Seq() ): Iterator[ColumnarBatch] = { - assert( + require( inputPartition.isInstanceOf[GlutenPartition], "CH backend only accepts GlutenPartition in GlutenWholeStageColumnarRDD.") - - val transKernel = new CHNativeExpressionEvaluator() - val inBatchIters = new JArrayList[GeneralInIterator](inputIterators.map { - iter => new ColumnarNativeIterator(CHIteratorApi.genCloseableColumnBatchIterator(iter).asJava) - }.asJava) - val splitInfoByteArray = inputPartition .asInstanceOf[GlutenPartition] .splitInfosByteArray - val nativeIter = - transKernel.createKernelWithBatchIterator( - inputPartition.plan, - splitInfoByteArray, - inBatchIters, - false) + val wsPlan = inputPartition.plan + val materializeInput = false - val iter = new CollectMetricIterator( - nativeIter, - updateNativeMetrics, - updateInputMetrics, - context.taskMetrics().inputMetrics) - - context.addTaskFailureListener( - (ctx, _) => { - if (ctx.isInterrupted()) { - iter.cancel() - } - }) - context.addTaskCompletionListener[Unit](_ => iter.close()) - - // TODO: SPARK-25083 remove the type erasure hack in data source scan new InterruptibleIterator( context, - new CloseableCHColumnBatchIterator( - iter.asInstanceOf[Iterator[ColumnarBatch]], - Some(pipelineTime))) + createCloseIterator( + context, + pipelineTime, + updateNativeMetrics, + Some(updateInputMetrics), + createNativeIterator(splitInfoByteArray, wsPlan, materializeInput, inputIterators)) + ) } // Generate Iterator[ColumnarBatch] for final stage. @@ -252,52 +277,26 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { partitionIndex: Int, materializeInput: Boolean): Iterator[ColumnarBatch] = { // scalastyle:on argcount - GlutenConfig.getConf - - val transKernel = new CHNativeExpressionEvaluator() - val columnarNativeIterator = - new JArrayList[GeneralInIterator](inputIterators.map { - iter => - new ColumnarNativeIterator(CHIteratorApi.genCloseableColumnBatchIterator(iter).asJava) - }.asJava) - // we need to complete dependency RDD's firstly - val nativeIterator = transKernel.createKernelWithBatchIterator( - rootNode.toProtobuf.toByteArray, - // Final iterator does not contain scan split, so pass empty split info to native here. - new Array[Array[Byte]](0), - columnarNativeIterator, - materializeInput - ) - - val iter = new CollectMetricIterator(nativeIterator, updateNativeMetrics, null, null) - context.addTaskFailureListener( - (ctx, _) => { - if (ctx.isInterrupted()) { - iter.cancel() - } - }) - context.addTaskCompletionListener[Unit](_ => iter.close()) - new CloseableCHColumnBatchIterator(iter, Some(pipelineTime)) - } -} + // Final iterator does not contain scan split, so pass empty split info to native here. + val splitInfoByteArray = new Array[Array[Byte]](0) + val wsPlan = rootNode.toProtobuf.toByteArray -object CHIteratorApi { - - /** Generate closeable ColumnBatch iterator. */ - def genCloseableColumnBatchIterator(iter: Iterator[ColumnarBatch]): Iterator[ColumnarBatch] = { - iter match { - case _: CloseableCHColumnBatchIterator => iter - case _ => new CloseableCHColumnBatchIterator(iter) - } + // we need to complete dependency RDD's firstly + createCloseIterator( + context, + pipelineTime, + updateNativeMetrics, + None, + createNativeIterator(splitInfoByteArray, wsPlan, materializeInput, inputIterators)) } } class CollectMetricIterator( val nativeIterator: BatchIterator, val updateNativeMetrics: IMetrics => Unit, - val updateInputMetrics: InputMetricsWrapper => Unit, - val inputMetrics: InputMetrics + val updateInputMetrics: Option[InputMetricsWrapper => Unit] = None, + val inputMetrics: InputMetrics = null ) extends Iterator[ColumnarBatch] { private var outputRowCount = 0L private var outputVectorCount = 0L @@ -329,9 +328,7 @@ class CollectMetricIterator( val nativeMetrics = nativeIterator.getMetrics.asInstanceOf[NativeMetrics] nativeMetrics.setFinalOutputMetrics(outputRowCount, outputVectorCount) updateNativeMetrics(nativeMetrics) - if (updateInputMetrics != null) { - updateInputMetrics(inputMetrics) - } + updateInputMetrics.foreach(_(inputMetrics)) metricsUpdated = true } } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 1c83e326eed4..ac3ea61ff810 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -50,7 +50,6 @@ import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.AQEShuffleReadExec import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation} -import org.apache.spark.sql.execution.datasources.GlutenWriterColumnarRules.NativeWritePostRule import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} @@ -583,14 +582,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { override def genExtendedColumnarTransformRules(): List[SparkSession => Rule[SparkPlan]] = List() - /** - * Generate extended columnar post-rules. - * - * @return - */ - override def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] = - List(spark => NativeWritePostRule(spark)) - override def genInjectPostHocResolutionRules(): List[SparkSession => Rule[LogicalPlan]] = { List() } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala index a7e7769e7736..da9d9c7586c0 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/CHHashJoinExecTransformer.scala @@ -16,7 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.backendsapi.clickhouse.CHIteratorApi import org.apache.gluten.extension.ValidationResult import org.apache.gluten.utils.{BroadcastHashJoinStrategy, CHJoinValidateUtil, ShuffleHashJoinStrategy} @@ -75,7 +74,7 @@ case class CHBroadcastBuildSideRDD( override def genBroadcastBuildSideIterator(): Iterator[ColumnarBatch] = { CHBroadcastBuildSideCache.getOrBuildBroadcastHashTable(broadcasted, broadcastContext) - CHIteratorApi.genCloseableColumnBatchIterator(Iterator.empty) + Iterator.empty } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala index 9269303d9251..ccf7bb5d5b2a 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala @@ -21,6 +21,7 @@ import org.apache.gluten.execution.AllDataTypesWithComplexType.genTestData import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf +import org.apache.spark.gluten.NativeWriteChecker import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper @@ -28,11 +29,14 @@ import org.apache.spark.sql.test.SharedSparkSession import org.scalatest.BeforeAndAfterAll +import scala.reflect.runtime.universe.TypeTag + class GlutenClickHouseNativeWriteTableSuite extends GlutenClickHouseWholeStageTransformerSuite with AdaptiveSparkPlanHelper with SharedSparkSession - with BeforeAndAfterAll { + with BeforeAndAfterAll + with NativeWriteChecker { private var _hiveSpark: SparkSession = _ @@ -114,16 +118,19 @@ class GlutenClickHouseNativeWriteTableSuite def getColumnName(s: String): String = { s.replaceAll("\\(", "_").replaceAll("\\)", "_") } + import collection.immutable.ListMap import java.io.File def writeIntoNewTableWithSql(table_name: String, table_create_sql: String)( fields: Seq[String]): Unit = { - spark.sql(table_create_sql) - spark.sql( - s"insert overwrite $table_name select ${fields.mkString(",")}" + - s" from origin_table") + withDestinationTable(table_name, table_create_sql) { + checkNativeWrite( + s"insert overwrite $table_name select ${fields.mkString(",")}" + + s" from origin_table", + checkNative = true) + } } def writeAndCheckRead( @@ -170,82 +177,86 @@ class GlutenClickHouseNativeWriteTableSuite }) } - test("test insert into dir") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val originDF = spark.createDataFrame(genTestData()) - originDF.createOrReplaceTempView("origin_table") + private val fields_ = ListMap( + ("string_field", "string"), + ("int_field", "int"), + ("long_field", "long"), + ("float_field", "float"), + ("double_field", "double"), + ("short_field", "short"), + ("byte_field", "byte"), + ("boolean_field", "boolean"), + ("decimal_field", "decimal(23,12)"), + ("date_field", "date") + ) - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) + def withDestinationTable(table: String, createTableSql: String)(f: => Unit): Unit = { + spark.sql(s"drop table IF EXISTS $table") + spark.sql(s"$createTableSql") + f + } - for (format <- formats) { - spark.sql( - s"insert overwrite local directory '$basePath/test_insert_into_${format}_dir1' " - + s"stored as $format select " - + fields.keys.mkString(",") + - " from origin_table cluster by (byte_field)") - spark.sql( - s"insert overwrite local directory '$basePath/test_insert_into_${format}_dir2' " + - s"stored as $format " + - "select string_field, sum(int_field) as x from origin_table group by string_field") - } + def nativeWrite(f: String => Unit): Unit = { + withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { + formats.foreach(f(_)) } } - test("test insert into partition") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.sql.orc.compression.codec", "lz4"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val originDF = spark.createDataFrame(genTestData()) - originDF.createOrReplaceTempView("origin_table") - - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) - - for (format <- formats) { - val table_name = table_name_template.format(format) - spark.sql(s"drop table IF EXISTS $table_name") + def nativeWrite2( + f: String => (String, String, String), + extraCheck: (String, String, String) => Unit = null): Unit = nativeWrite { + format => + val (table_name, table_create_sql, insert_sql) = f(format) + withDestinationTable(table_name, table_create_sql) { + checkNativeWrite(insert_sql, checkNative = true) + Option(extraCheck).foreach(_(table_name, table_create_sql, insert_sql)) + } + } - val table_create_sql = - s"create table if not exists $table_name (" + - fields - .map(f => s"${f._1} ${f._2}") - .mkString(",") + - " ) partitioned by (another_date_field date) " + - s"stored as $format" + def nativeWriteWithOriginalView[A <: Product: TypeTag]( + data: Seq[A], + viewName: String, + pairs: (String, String)*)(f: String => Unit): Unit = { + val configs = pairs :+ ("spark.gluten.sql.native.writer.enabled", "true") + withSQLConf(configs: _*) { + withTempView(viewName) { + spark.createDataFrame(data).createOrReplaceTempView(viewName) + formats.foreach(f(_)) + } + } + } - spark.sql(table_create_sql) + test("test insert into dir") { + nativeWriteWithOriginalView(genTestData(), "origin_table") { + format => + Seq( + s"""insert overwrite local directory '$basePath/test_insert_into_${format}_dir1' + |stored as $format select ${fields_.keys.mkString(",")} + |from origin_table""".stripMargin, + s"""insert overwrite local directory '$basePath/test_insert_into_${format}_dir2' + |stored as $format select string_field, sum(int_field) as x + |from origin_table group by string_field""".stripMargin + ).foreach(checkNativeWrite(_, checkNative = true)) + } + } - spark.sql( - s"insert into $table_name partition(another_date_field = '2020-01-01') select " - + fields.keys.mkString(",") + - " from origin_table") + test("test insert into partition") { + def destination(format: String): (String, String, String) = { + val table_name = table_name_template.format(format) + val table_create_sql = + s"""create table if not exists $table_name + |(${fields_.map(f => s"${f._1} ${f._2}").mkString(",")}) + |partitioned by (another_date_field date) stored as $format""".stripMargin + val insert_sql = + s"""insert into $table_name partition(another_date_field = '2020-01-01') + | select ${fields_.keys.mkString(",")} from origin_table""".stripMargin + (table_name, table_create_sql, insert_sql) + } + def nativeFormatWrite(format: String): Unit = { + val (table_name, table_create_sql, insert_sql) = destination(format) + withDestinationTable(table_name, table_create_sql) { + checkNativeWrite(insert_sql, checkNative = true) var files = recursiveListFiles(new File(getWarehouseDir + "/" + table_name)) .filter(_.getName.endsWith(s".$format")) if (format == "orc") { @@ -255,154 +266,103 @@ class GlutenClickHouseNativeWriteTableSuite assert(files.head.getAbsolutePath.contains("another_date_field=2020-01-01")) } } + + nativeWriteWithOriginalView( + genTestData(), + "origin_table", + ("spark.sql.orc.compression.codec", "lz4"))(nativeFormatWrite) } test("test CTAS") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val originDF = spark.createDataFrame(genTestData()) - originDF.createOrReplaceTempView("origin_table") - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) - - for (format <- formats) { + nativeWriteWithOriginalView(genTestData(), "origin_table") { + format => val table_name = table_name_template.format(format) - spark.sql(s"drop table IF EXISTS $table_name") val table_create_sql = s"create table $table_name using $format as select " + - fields + fields_ .map(f => s"${f._1}") .mkString(",") + " from origin_table" - spark.sql(table_create_sql) - spark.sql(s"drop table IF EXISTS $table_name") + val insert_sql = + s"create table $table_name as select " + + fields_ + .map(f => s"${f._1}") + .mkString(",") + + " from origin_table" + withDestinationTable(table_name, table_create_sql) { + spark.sql(s"drop table IF EXISTS $table_name") - try { - val table_create_sql = - s"create table $table_name as select " + - fields - .map(f => s"${f._1}") - .mkString(",") + - " from origin_table" - spark.sql(table_create_sql) - } catch { - case _: UnsupportedOperationException => // expected - case _: Exception => fail("should not throw exception") + try { + // FIXME: using checkNativeWrite + spark.sql(insert_sql) + } catch { + case _: UnsupportedOperationException => // expected + case e: Exception => fail("should not throw exception", e) + } } - } } } test("test insert into partition, bigo's case which incur InsertIntoHiveTable") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - ("spark.sql.hive.convertMetastoreParquet", "false"), - ("spark.sql.hive.convertMetastoreOrc", "false"), - (GlutenConfig.GLUTEN_ENABLED.key, "true") - ) { - - val originDF = spark.createDataFrame(genTestData()) - originDF.createOrReplaceTempView("origin_table") - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) - - for (format <- formats) { - val table_name = table_name_template.format(format) - spark.sql(s"drop table IF EXISTS $table_name") - val table_create_sql = s"create table if not exists $table_name (" + fields - .map(f => s"${f._1} ${f._2}") - .mkString(",") + " ) partitioned by (another_date_field string)" + - s"stored as $format" + def destination(format: String): (String, String, String) = { + val table_name = table_name_template.format(format) + val table_create_sql = s"create table if not exists $table_name (" + fields_ + .map(f => s"${f._1} ${f._2}") + .mkString(",") + " ) partitioned by (another_date_field string)" + + s"stored as $format" + val insert_sql = + s"insert overwrite table $table_name " + + "partition(another_date_field = '2020-01-01') select " + + fields_.keys.mkString(",") + " from (select " + fields_.keys.mkString( + ",") + ", row_number() over (order by int_field desc) as rn " + + "from origin_table where float_field > 3 ) tt where rn <= 100" + (table_name, table_create_sql, insert_sql) + } - spark.sql(table_create_sql) - spark.sql( - s"insert overwrite table $table_name " + - "partition(another_date_field = '2020-01-01') select " - + fields.keys.mkString(",") + " from (select " + fields.keys.mkString( - ",") + ", row_number() over (order by int_field desc) as rn " + - "from origin_table where float_field > 3 ) tt where rn <= 100") + def nativeFormatWrite(format: String): Unit = { + val (table_name, table_create_sql, insert_sql) = destination(format) + withDestinationTable(table_name, table_create_sql) { + checkNativeWrite(insert_sql, checkNative = true) val files = recursiveListFiles(new File(getWarehouseDir + "/" + table_name)) .filter(_.getName.startsWith("part")) assert(files.length == 1) assert(files.head.getAbsolutePath.contains("another_date_field=2020-01-01")) } } + + nativeWriteWithOriginalView( + genTestData(), + "origin_table", + ("spark.sql.hive.convertMetastoreParquet", "false"), + ("spark.sql.hive.convertMetastoreOrc", "false"))(nativeFormatWrite) } test("test 1-col partitioned table") { + nativeWrite { + format => + { + val table_name = table_name_template.format(format) + val table_create_sql = + s"create table if not exists $table_name (" + + fields_ + .filterNot(e => e._1.equals("date_field")) + .map(f => s"${f._1} ${f._2}") + .mkString(",") + + " ) partitioned by (date_field date) " + + s"stored as $format" - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) - - for (format <- formats) { - val table_name = table_name_template.format(format) - val table_create_sql = - s"create table if not exists $table_name (" + - fields - .filterNot(e => e._1.equals("date_field")) - .map(f => s"${f._1} ${f._2}") - .mkString(",") + - " ) partitioned by (date_field date) " + - s"stored as $format" - - writeAndCheckRead( - table_name, - writeIntoNewTableWithSql(table_name, table_create_sql), - fields.keys.toSeq) - } + writeAndCheckRead( + table_name, + writeIntoNewTableWithSql(table_name, table_create_sql), + fields_.keys.toSeq) + } } } // even if disable native writer, this UT fail, spark bug??? ignore("test 1-col partitioned table, partitioned by already ordered column") { withSQLConf(("spark.gluten.sql.native.writer.enabled", "false")) { - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) val originDF = spark.createDataFrame(genTestData()) originDF.createOrReplaceTempView("origin_table") @@ -410,7 +370,7 @@ class GlutenClickHouseNativeWriteTableSuite val table_name = table_name_template.format(format) val table_create_sql = s"create table if not exists $table_name (" + - fields + fields_ .filterNot(e => e._1.equals("date_field")) .map(f => s"${f._1} ${f._2}") .mkString(",") + @@ -420,31 +380,27 @@ class GlutenClickHouseNativeWriteTableSuite spark.sql(s"drop table IF EXISTS $table_name") spark.sql(table_create_sql) spark.sql( - s"insert overwrite $table_name select ${fields.mkString(",")}" + + s"insert overwrite $table_name select ${fields_.mkString(",")}" + s" from origin_table order by date_field") } } } test("test 2-col partitioned table") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date"), - ("byte_field", "byte") - ) - - for (format <- formats) { + val fields: ListMap[String, String] = ListMap( + ("string_field", "string"), + ("int_field", "int"), + ("long_field", "long"), + ("float_field", "float"), + ("double_field", "double"), + ("short_field", "short"), + ("boolean_field", "boolean"), + ("decimal_field", "decimal(23,12)"), + ("date_field", "date"), + ("byte_field", "byte") + ) + nativeWrite { + format => val table_name = table_name_template.format(format) val table_create_sql = s"create table if not exists $table_name (" + @@ -458,7 +414,6 @@ class GlutenClickHouseNativeWriteTableSuite table_name, writeIntoNewTableWithSql(table_name, table_create_sql), fields.keys.toSeq) - } } } @@ -506,25 +461,21 @@ class GlutenClickHouseNativeWriteTableSuite // This test case will be failed with incorrect result randomly, ignore first. ignore("test hive parquet/orc table, all columns being partitioned. ") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val fields: ListMap[String, String] = ListMap( - ("date_field", "date"), - ("timestamp_field", "timestamp"), - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)") - ) - - for (format <- formats) { + val fields: ListMap[String, String] = ListMap( + ("date_field", "date"), + ("timestamp_field", "timestamp"), + ("string_field", "string"), + ("int_field", "int"), + ("long_field", "long"), + ("float_field", "float"), + ("double_field", "double"), + ("short_field", "short"), + ("byte_field", "byte"), + ("boolean_field", "boolean"), + ("decimal_field", "decimal(23,12)") + ) + nativeWrite { + format => val table_name = table_name_template.format(format) val table_create_sql = s"create table if not exists $table_name (" + @@ -540,20 +491,15 @@ class GlutenClickHouseNativeWriteTableSuite table_name, writeIntoNewTableWithSql(table_name, table_create_sql), fields.keys.toSeq) - } } } - test(("test hive parquet/orc table with aggregated results")) { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val fields: ListMap[String, String] = ListMap( - ("sum(int_field)", "bigint") - ) - - for (format <- formats) { + test("test hive parquet/orc table with aggregated results") { + val fields: ListMap[String, String] = ListMap( + ("sum(int_field)", "bigint") + ) + nativeWrite { + format => val table_name = table_name_template.format(format) val table_create_sql = s"create table if not exists $table_name (" + @@ -566,29 +512,12 @@ class GlutenClickHouseNativeWriteTableSuite table_name, writeIntoNewTableWithSql(table_name, table_create_sql), fields.keys.toSeq) - } } } test("test 1-col partitioned + 1-col bucketed table") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - - val fields: ListMap[String, String] = ListMap( - ("string_field", "string"), - ("int_field", "int"), - ("long_field", "long"), - ("float_field", "float"), - ("double_field", "double"), - ("short_field", "short"), - ("byte_field", "byte"), - ("boolean_field", "boolean"), - ("decimal_field", "decimal(23,12)"), - ("date_field", "date") - ) - - for (format <- formats) { + nativeWrite { + format => // spark write does not support bucketed table // https://issues.apache.org/jira/browse/SPARK-19256 val table_name = table_name_template.format(format) @@ -604,7 +533,7 @@ class GlutenClickHouseNativeWriteTableSuite .bucketBy(2, "byte_field") .saveAsTable(table_name) }, - fields.keys.toSeq + fields_.keys.toSeq ) assert( @@ -614,10 +543,8 @@ class GlutenClickHouseNativeWriteTableSuite .filter(!_.getName.equals("date_field=__HIVE_DEFAULT_PARTITION__")) .head .listFiles() - .filter(!_.isHidden) - .length == 2 + .count(!_.isHidden) == 2 ) // 2 bucket files - } } } @@ -745,8 +672,8 @@ class GlutenClickHouseNativeWriteTableSuite } test("test consecutive blocks having same partition value") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { + nativeWrite { + format => val table_name = table_name_template.format(format) spark.sql(s"drop table IF EXISTS $table_name") @@ -760,15 +687,14 @@ class GlutenClickHouseNativeWriteTableSuite .partitionBy("p") .saveAsTable(table_name) - val ret = spark.sql("select sum(id) from " + table_name).collect().apply(0).apply(0) + val ret = spark.sql(s"select sum(id) from $table_name").collect().apply(0).apply(0) assert(ret == 449985000) - } } } test("test decimal with rand()") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { + nativeWrite { + format => val table_name = table_name_template.format(format) spark.sql(s"drop table IF EXISTS $table_name") spark @@ -778,32 +704,30 @@ class GlutenClickHouseNativeWriteTableSuite .format(format) .partitionBy("p") .saveAsTable(table_name) - val ret = spark.sql("select max(p) from " + table_name).collect().apply(0).apply(0) - } + val ret = spark.sql(s"select max(p) from $table_name").collect().apply(0).apply(0) } } test("test partitioned by constant") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { - spark.sql(s"drop table IF EXISTS tmp_123_$format") - spark.sql( - s"create table tmp_123_$format(" + - s"x1 string, x2 bigint,x3 string, x4 bigint, x5 string )" + - s"partitioned by (day date) stored as $format") - - spark.sql( - s"insert into tmp_123_$format partition(day) " + - "select cast(id as string), id, cast(id as string), id, cast(id as string), " + - "'2023-05-09' from range(10000000)") - } + nativeWrite2 { + format => + val table_name = s"tmp_123_$format" + val create_sql = + s"""create table tmp_123_$format( + |x1 string, x2 bigint,x3 string, x4 bigint, x5 string ) + |partitioned by (day date) stored as $format""".stripMargin + val insert_sql = + s"""insert into tmp_123_$format partition(day) + |select cast(id as string), id, cast(id as string), + | id, cast(id as string), '2023-05-09' + |from range(10000000)""".stripMargin + (table_name, create_sql, insert_sql) } } test("test bucketed by constant") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - - for (format <- formats) { + nativeWrite { + format => val table_name = table_name_template.format(format) spark.sql(s"drop table IF EXISTS $table_name") @@ -815,15 +739,13 @@ class GlutenClickHouseNativeWriteTableSuite .bucketBy(2, "p") .saveAsTable(table_name) - val ret = spark.sql("select count(*) from " + table_name).collect().apply(0).apply(0) - } + assertResult(10000000)(spark.table(table_name).count()) } } test("test consecutive null values being partitioned") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - - for (format <- formats) { + nativeWrite { + format => val table_name = table_name_template.format(format) spark.sql(s"drop table IF EXISTS $table_name") @@ -835,14 +757,13 @@ class GlutenClickHouseNativeWriteTableSuite .partitionBy("p") .saveAsTable(table_name) - val ret = spark.sql("select count(*) from " + table_name).collect().apply(0).apply(0) - } + assertResult(30000)(spark.table(table_name).count()) } } test("test consecutive null values being bucketed") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { + nativeWrite { + format => val table_name = table_name_template.format(format) spark.sql(s"drop table IF EXISTS $table_name") @@ -854,78 +775,79 @@ class GlutenClickHouseNativeWriteTableSuite .bucketBy(2, "p") .saveAsTable(table_name) - val ret = spark.sql("select count(*) from " + table_name).collect().apply(0).apply(0) - } + assertResult(30000)(spark.table(table_name).count()) } } test("test native write with empty dataset") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { + nativeWrite2( + format => { val table_name = "t_" + format - spark.sql(s"drop table IF EXISTS $table_name") - spark.sql(s"create table $table_name (id int, str string) stored as $format") - spark.sql( - s"insert into $table_name select id, cast(id as string) from range(10)" + - " where id > 100") + ( + table_name, + s"create table $table_name (id int, str string) stored as $format", + s"insert into $table_name select id, cast(id as string) from range(10) where id > 100" + ) + }, + (table_name, _, _) => { + assertResult(0)(spark.table(table_name).count()) } - } + ) } test("test native write with union") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { + nativeWrite { + format => val table_name = "t_" + format - spark.sql(s"drop table IF EXISTS $table_name") - spark.sql(s"create table $table_name (id int, str string) stored as $format") - spark.sql( - s"insert overwrite table $table_name " + - "select id, cast(id as string) from range(10) union all " + - "select 10, '10' from range(10)") - spark.sql( - s"insert overwrite table $table_name " + - "select id, cast(id as string) from range(10) union all " + - "select 10, cast(id as string) from range(10)") - - } + withDestinationTable( + table_name, + s"create table $table_name (id int, str string) stored as $format") { + checkNativeWrite( + s"insert overwrite table $table_name " + + "select id, cast(id as string) from range(10) union all " + + "select 10, '10' from range(10)", + checkNative = true) + checkNativeWrite( + s"insert overwrite table $table_name " + + "select id, cast(id as string) from range(10) union all " + + "select 10, cast(id as string) from range(10)", + checkNative = true + ) + } } } test("test native write and non-native read consistency") { - withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) { - for (format <- formats) { - val table_name = "t_" + format - spark.sql(s"drop table IF EXISTS $table_name") - spark.sql(s"create table $table_name (id int, name string, info char(4)) stored as $format") - spark.sql( - s"insert overwrite table $table_name " + - "select id, cast(id as string), concat('aaa', cast(id as string)) from range(10)") + nativeWrite2( + { + format => + val table_name = "t_" + format + ( + table_name, + s"create table $table_name (id int, name string, info char(4)) stored as $format", + s"insert overwrite table $table_name " + + "select id, cast(id as string), concat('aaa', cast(id as string)) from range(10)" + ) + }, + (table_name, _, _) => compareResultsAgainstVanillaSpark( s"select * from $table_name", compareResult = true, _ => {}) - } - } + ) } test("GLUTEN-4316: fix crash on dynamic partition inserting") { - withSQLConf( - ("spark.gluten.sql.native.writer.enabled", "true"), - (GlutenConfig.GLUTEN_ENABLED.key, "true")) { - formats.foreach( - format => { - val tbl = "t_" + format - spark.sql(s"drop table IF EXISTS $tbl") - val sql1 = - s"create table $tbl(a int, b map, c struct) " + - s"partitioned by (day string) stored as $format" - val sql2 = s"insert overwrite $tbl partition (day) " + - s"select id as a, str_to_map(concat('t1:','a','&t2:','b'),'&',':'), " + - s"struct('1', null) as c, '2024-01-08' as day from range(10)" - spark.sql(sql1) - spark.sql(sql2) - }) + nativeWrite2 { + format => + val tbl = "t_" + format + val sql1 = + s"create table $tbl(a int, b map, c struct) " + + s"partitioned by (day string) stored as $format" + val sql2 = s"insert overwrite $tbl partition (day) " + + s"select id as a, str_to_map(concat('t1:','a','&t2:','b'),'&',':'), " + + s"struct('1', null) as c, '2024-01-08' as day from range(10)" + (tbl, sql1, sql2) } } - } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala index 09fa3ff109f2..1b3df81667a0 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala @@ -46,7 +46,7 @@ class GlutenClickHouseTPCHMetricsSuite extends GlutenClickHouseTPCHAbstractSuite .set("spark.io.compression.codec", "LZ4") .set("spark.sql.shuffle.partitions", "1") .set("spark.sql.autoBroadcastJoinThreshold", "10MB") - .set("spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level", "DEBUG") + // .set("spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level", "DEBUG") .set( "spark.gluten.sql.columnar.backend.ch.runtime_settings.input_format_parquet_max_block_size", s"$parquetMaxBlockSize") diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala b/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala new file mode 100644 index 000000000000..79616d52d0bc --- /dev/null +++ b/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.gluten + +import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite + +import org.apache.spark.sql.execution.QueryExecution +import org.apache.spark.sql.execution.datasources.FakeRowAdaptor +import org.apache.spark.sql.util.QueryExecutionListener + +trait NativeWriteChecker extends GlutenClickHouseWholeStageTransformerSuite { + + def checkNativeWrite(sqlStr: String, checkNative: Boolean): Unit = { + var nativeUsed = false + + val queryListener = new QueryExecutionListener { + override def onFailure(f: String, qe: QueryExecution, e: Exception): Unit = {} + override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = { + if (!nativeUsed) { + nativeUsed = if (isSparkVersionGE("3.4")) { + false + } else { + qe.executedPlan.find(_.isInstanceOf[FakeRowAdaptor]).isDefined + } + } + } + } + + try { + spark.listenerManager.register(queryListener) + spark.sql(sqlStr) + spark.sparkContext.listenerBus.waitUntilEmpty() + assertResult(checkNative)(nativeUsed) + } finally { + spark.listenerManager.unregister(queryListener) + } + } +} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 1f868c4c2044..7b8d523a6d27 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -827,15 +827,6 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { buf.result } - /** - * Generate extended columnar post-rules. - * - * @return - */ - override def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] = { - SparkShimLoader.getSparkShims.getExtendedColumnarPostRules() ::: List() - } - override def genInjectPostHocResolutionRules(): List[SparkSession => Rule[LogicalPlan]] = { List(ArrowConvertorRule) } diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 94cd38003bad..ae3f6dbd5208 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -77,6 +77,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int UNKNOWN_TYPE; +extern const int CANNOT_PARSE_PROTOBUF_SCHEMA; } } @@ -466,17 +467,17 @@ String QueryPipelineUtil::explainPipeline(DB::QueryPipeline & pipeline) using namespace DB; -std::map BackendInitializerUtil::getBackendConfMap(std::string * plan) +std::map BackendInitializerUtil::getBackendConfMap(const std::string & plan) { std::map ch_backend_conf; - if (plan == nullptr) + if (plan.empty()) return ch_backend_conf; /// Parse backend configs from plan extensions do { auto plan_ptr = std::make_unique(); - auto success = plan_ptr->ParseFromString(*plan); + auto success = plan_ptr->ParseFromString(plan); if (!success) break; @@ -841,14 +842,8 @@ void BackendInitializerUtil::initCompiledExpressionCache(DB::Context::Configurat #endif } -void BackendInitializerUtil::init_json(std::string * plan_json) -{ - auto plan_ptr = std::make_unique(); - google::protobuf::util::JsonStringToMessage(plan_json->c_str(), plan_ptr.get()); - return init(new String(plan_ptr->SerializeAsString())); -} -void BackendInitializerUtil::init(std::string * plan) +void BackendInitializerUtil::init(const std::string & plan) { std::map backend_conf_map = getBackendConfMap(plan); DB::Context::ConfigurationPtr config = initConfig(backend_conf_map); @@ -906,7 +901,7 @@ void BackendInitializerUtil::init(std::string * plan) }); } -void BackendInitializerUtil::updateConfig(const DB::ContextMutablePtr & context, std::string * plan) +void BackendInitializerUtil::updateConfig(const DB::ContextMutablePtr & context, const std::string & plan) { std::map backend_conf_map = getBackendConfMap(plan); diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 94e0f0168e11..245d7b3d15c4 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -137,9 +137,8 @@ class BackendInitializerUtil /// Initialize two kinds of resources /// 1. global level resources like global_context/shared_context, notice that they can only be initialized once in process lifetime /// 2. session level resources like settings/configs, they can be initialized multiple times following the lifetime of executor/driver - static void init(std::string * plan); - static void init_json(std::string * plan_json); - static void updateConfig(const DB::ContextMutablePtr &, std::string *); + static void init(const std::string & plan); + static void updateConfig(const DB::ContextMutablePtr &, const std::string &); // use excel text parser @@ -196,7 +195,7 @@ class BackendInitializerUtil static void updateNewSettings(const DB::ContextMutablePtr &, const DB::Settings &); - static std::map getBackendConfMap(std::string * plan); + static std::map getBackendConfMap(const std::string & plan); inline static std::once_flag init_flag; inline static Poco::Logger * logger; @@ -283,10 +282,7 @@ class ConcurrentDeque return deq.empty(); } - std::deque unsafeGet() - { - return deq; - } + std::deque unsafeGet() { return deq; } private: std::deque deq; diff --git a/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp b/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp index 2b4eb824a5fd..5bb66e4b3f9d 100644 --- a/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp +++ b/cpp-ch/local-engine/Parser/CHColumnToSparkRow.cpp @@ -453,7 +453,7 @@ std::unique_ptr CHColumnToSparkRow::convertCHColumnToSparkRow(cons if (!block.columns()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "A block with empty columns"); std::unique_ptr spark_row_info = std::make_unique(block, masks); - spark_row_info->setBufferAddress(reinterpret_cast(alloc(spark_row_info->getTotalBytes(), 64))); + spark_row_info->setBufferAddress(static_cast(alloc(spark_row_info->getTotalBytes(), 64))); // spark_row_info->setBufferAddress(alignedAlloc(spark_row_info->getTotalBytes(), 64)); memset(spark_row_info->getBufferAddress(), 0, spark_row_info->getTotalBytes()); for (auto col_idx = 0; col_idx < spark_row_info->getNumCols(); col_idx++) diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 70db692c8009..3115950cdf09 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -87,14 +87,14 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; - extern const int UNKNOWN_TYPE; - extern const int BAD_ARGUMENTS; - extern const int NO_SUCH_DATA_PART; - extern const int UNKNOWN_FUNCTION; - extern const int CANNOT_PARSE_PROTOBUF_SCHEMA; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int INVALID_JOIN_ON_EXPRESSION; +extern const int LOGICAL_ERROR; +extern const int UNKNOWN_TYPE; +extern const int BAD_ARGUMENTS; +extern const int NO_SUCH_DATA_PART; +extern const int UNKNOWN_FUNCTION; +extern const int CANNOT_PARSE_PROTOBUF_SCHEMA; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int INVALID_JOIN_ON_EXPRESSION; } } @@ -144,16 +144,13 @@ void SerializedPlanParser::parseExtensions( if (extension.has_extension_function()) { function_mapping.emplace( - std::to_string(extension.extension_function().function_anchor()), - extension.extension_function().name()); + std::to_string(extension.extension_function().function_anchor()), extension.extension_function().name()); } } } std::shared_ptr SerializedPlanParser::expressionsToActionsDAG( - const std::vector & expressions, - const Block & header, - const Block & read_schema) + const std::vector & expressions, const Block & header, const Block & read_schema) { auto actions_dag = std::make_shared(blockToNameAndTypeList(header)); NamesWithAliases required_columns; @@ -259,8 +256,8 @@ std::string getDecimalFunction(const substrait::Type_Decimal & decimal, bool nul bool SerializedPlanParser::isReadRelFromJava(const substrait::ReadRel & rel) { - return rel.has_local_files() && rel.local_files().items().size() == 1 && rel.local_files().items().at(0).uri_file().starts_with( - "iterator"); + return rel.has_local_files() && rel.local_files().items().size() == 1 + && rel.local_files().items().at(0).uri_file().starts_with("iterator"); } bool SerializedPlanParser::isReadFromMergeTree(const substrait::ReadRel & rel) @@ -380,13 +377,13 @@ DataTypePtr wrapNullableType(bool nullable, DataTypePtr nested_type) return nested_type; } -QueryPlanPtr SerializedPlanParser::parse(std::unique_ptr plan) +QueryPlanPtr SerializedPlanParser::parse(const substrait::Plan & plan) { - logDebugMessage(*plan, "substrait plan"); - parseExtensions(plan->extensions()); - if (plan->relations_size() == 1) + logDebugMessage(plan, "substrait plan"); + parseExtensions(plan.extensions()); + if (plan.relations_size() == 1) { - auto root_rel = plan->relations().at(0); + auto root_rel = plan.relations().at(0); if (!root_rel.has_root()) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "must have root rel!"); @@ -587,9 +584,7 @@ SerializedPlanParser::getFunctionName(const std::string & function_signature, co { if (args.size() != 2) throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Spark function extract requires two args, function:{}", - function.ShortDebugString()); + ErrorCodes::BAD_ARGUMENTS, "Spark function extract requires two args, function:{}", function.ShortDebugString()); // Get the first arg: field const auto & extract_field = args.at(0); @@ -705,9 +700,7 @@ void SerializedPlanParser::parseArrayJoinArguments( /// The argument number of arrayJoin(converted from Spark explode/posexplode) should be 1 if (scalar_function.arguments_size() != 1) throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Argument number of arrayJoin should be 1 instead of {}", - scalar_function.arguments_size()); + ErrorCodes::BAD_ARGUMENTS, "Argument number of arrayJoin should be 1 instead of {}", scalar_function.arguments_size()); auto function_name_copy = function_name; parseFunctionArguments(actions_dag, parsed_args, function_name_copy, scalar_function); @@ -746,11 +739,7 @@ void SerializedPlanParser::parseArrayJoinArguments( } ActionsDAG::NodeRawConstPtrs SerializedPlanParser::parseArrayJoinWithDAG( - const substrait::Expression & rel, - std::vector & result_names, - ActionsDAGPtr actions_dag, - bool keep_result, - bool position) + const substrait::Expression & rel, std::vector & result_names, ActionsDAGPtr actions_dag, bool keep_result, bool position) { if (!rel.has_scalar_function()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The root of expression should be a scalar function:\n {}", rel.DebugString()); @@ -774,7 +763,8 @@ ActionsDAG::NodeRawConstPtrs SerializedPlanParser::parseArrayJoinWithDAG( auto tuple_element_builder = FunctionFactory::instance().get("sparkTupleElement", context); auto tuple_index_type = std::make_shared(); - auto add_tuple_element = [&](const ActionsDAG::Node * tuple_node, size_t i) -> const ActionsDAG::Node * { + auto add_tuple_element = [&](const ActionsDAG::Node * tuple_node, size_t i) -> const ActionsDAG::Node * + { ColumnWithTypeAndName index_col(tuple_index_type->createColumnConst(1, i), tuple_index_type, getUniqueName(std::to_string(i))); const auto * index_node = &actions_dag->addColumn(std::move(index_col)); auto result_name = "sparkTupleElement(" + tuple_node->result_name + ", " + index_node->result_name + ")"; @@ -866,10 +856,7 @@ ActionsDAG::NodeRawConstPtrs SerializedPlanParser::parseArrayJoinWithDAG( } const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( - const substrait::Expression & rel, - std::string & result_name, - ActionsDAGPtr actions_dag, - bool keep_result) + const substrait::Expression & rel, std::string & result_name, ActionsDAGPtr actions_dag, bool keep_result) { if (!rel.has_scalar_function()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "the root of expression should be a scalar function:\n {}", rel.DebugString()); @@ -884,10 +871,7 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( if (auto func_parser = FunctionParserFactory::instance().tryGet(func_name, this)) { LOG_DEBUG( - &Poco::Logger::get("SerializedPlanParser"), - "parse function {} by function parser: {}", - func_name, - func_parser->getName()); + &Poco::Logger::get("SerializedPlanParser"), "parse function {} by function parser: {}", func_name, func_parser->getName()); const auto * result_node = func_parser->parse(scalar_function, actions_dag); if (keep_result) actions_dag->addOrReplaceInOutputs(*result_node); @@ -956,12 +940,10 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( UInt32 precision = rel.scalar_function().output_type().decimal().precision(); UInt32 scale = rel.scalar_function().output_type().decimal().scale(); auto uint32_type = std::make_shared(); - new_args.emplace_back( - &actions_dag->addColumn( - ColumnWithTypeAndName(uint32_type->createColumnConst(1, precision), uint32_type, getUniqueName(toString(precision))))); - new_args.emplace_back( - &actions_dag->addColumn( - ColumnWithTypeAndName(uint32_type->createColumnConst(1, scale), uint32_type, getUniqueName(toString(scale))))); + new_args.emplace_back(&actions_dag->addColumn( + ColumnWithTypeAndName(uint32_type->createColumnConst(1, precision), uint32_type, getUniqueName(toString(precision))))); + new_args.emplace_back(&actions_dag->addColumn( + ColumnWithTypeAndName(uint32_type->createColumnConst(1, scale), uint32_type, getUniqueName(toString(scale))))); args = std::move(new_args); } else if (startsWith(function_signature, "make_decimal:")) @@ -976,12 +958,10 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( UInt32 precision = rel.scalar_function().output_type().decimal().precision(); UInt32 scale = rel.scalar_function().output_type().decimal().scale(); auto uint32_type = std::make_shared(); - new_args.emplace_back( - &actions_dag->addColumn( - ColumnWithTypeAndName(uint32_type->createColumnConst(1, precision), uint32_type, getUniqueName(toString(precision))))); - new_args.emplace_back( - &actions_dag->addColumn( - ColumnWithTypeAndName(uint32_type->createColumnConst(1, scale), uint32_type, getUniqueName(toString(scale))))); + new_args.emplace_back(&actions_dag->addColumn( + ColumnWithTypeAndName(uint32_type->createColumnConst(1, precision), uint32_type, getUniqueName(toString(precision))))); + new_args.emplace_back(&actions_dag->addColumn( + ColumnWithTypeAndName(uint32_type->createColumnConst(1, scale), uint32_type, getUniqueName(toString(scale))))); args = std::move(new_args); } @@ -999,9 +979,8 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( actions_dag, function_node, // as stated in isTypeMatched， currently we don't change nullability of the result type - function_node->result_type->isNullable() - ? local_engine::wrapNullableType(true, result_type)->getName() - : local_engine::removeNullable(result_type)->getName(), + function_node->result_type->isNullable() ? local_engine::wrapNullableType(true, result_type)->getName() + : local_engine::removeNullable(result_type)->getName(), function_node->result_name, CastType::accurateOrNull); } @@ -1011,9 +990,8 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG( actions_dag, function_node, // as stated in isTypeMatched， currently we don't change nullability of the result type - function_node->result_type->isNullable() - ? local_engine::wrapNullableType(true, result_type)->getName() - : local_engine::removeNullable(result_type)->getName(), + function_node->result_type->isNullable() ? local_engine::wrapNullableType(true, result_type)->getName() + : local_engine::removeNullable(result_type)->getName(), function_node->result_name); } } @@ -1159,9 +1137,7 @@ void SerializedPlanParser::parseFunctionArgument( } const ActionsDAG::Node * SerializedPlanParser::parseFunctionArgument( - ActionsDAGPtr & actions_dag, - const std::string & function_name, - const substrait::FunctionArgument & arg) + ActionsDAGPtr & actions_dag, const std::string & function_name, const substrait::FunctionArgument & arg) { const ActionsDAG::Node * res; if (arg.value().has_scalar_function()) @@ -1189,11 +1165,8 @@ std::pair SerializedPlanParser::convertStructFieldType(const } auto type_id = type->getTypeId(); - if (type_id == TypeIndex::UInt8 || type_id == TypeIndex::UInt16 || type_id == TypeIndex::UInt32 - || type_id == TypeIndex::UInt64) - { + if (type_id == TypeIndex::UInt8 || type_id == TypeIndex::UInt16 || type_id == TypeIndex::UInt32 || type_id == TypeIndex::UInt64) return {type, field}; - } UINT_CONVERT(type, field, Int8) UINT_CONVERT(type, field, Int16) UINT_CONVERT(type, field, Int32) @@ -1203,11 +1176,7 @@ std::pair SerializedPlanParser::convertStructFieldType(const } ActionsDAGPtr SerializedPlanParser::parseFunction( - const Block & header, - const substrait::Expression & rel, - std::string & result_name, - ActionsDAGPtr actions_dag, - bool keep_result) + const Block & header, const substrait::Expression & rel, std::string & result_name, ActionsDAGPtr actions_dag, bool keep_result) { if (!actions_dag) actions_dag = std::make_shared(blockToNameAndTypeList(header)); @@ -1217,11 +1186,7 @@ ActionsDAGPtr SerializedPlanParser::parseFunction( } ActionsDAGPtr SerializedPlanParser::parseFunctionOrExpression( - const Block & header, - const substrait::Expression & rel, - std::string & result_name, - ActionsDAGPtr actions_dag, - bool keep_result) + const Block & header, const substrait::Expression & rel, std::string & result_name, ActionsDAGPtr actions_dag, bool keep_result) { if (!actions_dag) actions_dag = std::make_shared(blockToNameAndTypeList(header)); @@ -1303,7 +1268,8 @@ ActionsDAGPtr SerializedPlanParser::parseJsonTuple( = &actions_dag->addFunction(json_extract_builder, {json_expr_node, extract_expr_node}, json_extract_result_name); auto tuple_element_builder = FunctionFactory::instance().get("sparkTupleElement", context); auto tuple_index_type = std::make_shared(); - auto add_tuple_element = [&](const ActionsDAG::Node * tuple_node, size_t i) -> const ActionsDAG::Node * { + auto add_tuple_element = [&](const ActionsDAG::Node * tuple_node, size_t i) -> const ActionsDAG::Node * + { ColumnWithTypeAndName index_col(tuple_index_type->createColumnConst(1, i), tuple_index_type, getUniqueName(std::to_string(i))); const auto * index_node = &actions_dag->addColumn(std::move(index_col)); auto result_name = "sparkTupleElement(" + tuple_node->result_name + ", " + index_node->result_name + ")"; @@ -1528,9 +1494,7 @@ std::pair SerializedPlanParser::parseLiteral(const substrait } default: { throw Exception( - ErrorCodes::UNKNOWN_TYPE, - "Unsupported spark literal type {}", - magic_enum::enum_name(literal.literal_type_case())); + ErrorCodes::UNKNOWN_TYPE, "Unsupported spark literal type {}", magic_enum::enum_name(literal.literal_type_case())); } } return std::make_pair(std::move(type), std::move(field)); @@ -1732,8 +1696,7 @@ substrait::ReadRel::ExtensionTable SerializedPlanParser::parseExtensionTable(con { substrait::ReadRel::ExtensionTable extension_table; google::protobuf::io::CodedInputStream coded_in( - reinterpret_cast(split_info.data()), - static_cast(split_info.size())); + reinterpret_cast(split_info.data()), static_cast(split_info.size())); coded_in.SetRecursionLimit(100000); auto ok = extension_table.ParseFromCodedStream(&coded_in); @@ -1747,8 +1710,7 @@ substrait::ReadRel::LocalFiles SerializedPlanParser::parseLocalFiles(const std:: { substrait::ReadRel::LocalFiles local_files; google::protobuf::io::CodedInputStream coded_in( - reinterpret_cast(split_info.data()), - static_cast(split_info.size())); + reinterpret_cast(split_info.data()), static_cast(split_info.size())); coded_in.SetRecursionLimit(100000); auto ok = local_files.ParseFromCodedStream(&coded_in); @@ -1758,10 +1720,44 @@ substrait::ReadRel::LocalFiles SerializedPlanParser::parseLocalFiles(const std:: return local_files; } +std::unique_ptr SerializedPlanParser::createExecutor(DB::QueryPlanPtr query_plan) +{ + Stopwatch stopwatch; + auto * logger = &Poco::Logger::get("SerializedPlanParser"); + const Settings & settings = context->getSettingsRef(); + + QueryPriorities priorities; + auto query_status = std::make_shared( + context, + "", + context->getClientInfo(), + priorities.insert(static_cast(settings.priority)), + CurrentThread::getGroup(), + IAST::QueryKind::Select, + settings, + 0); + + QueryPlanOptimizationSettings optimization_settings{.optimize_plan = settings.query_plan_enable_optimizations}; + auto pipeline_builder = query_plan->buildQueryPipeline( + optimization_settings, + BuildQueryPipelineSettings{ + .actions_settings + = ExpressionActionsSettings{.can_compile_expressions = true, .min_count_to_compile_expression = 3, .compile_expressions = CompileExpressions::yes}, + .process_list_element = query_status}); + QueryPipeline pipeline = QueryPipelineBuilder::getPipeline(std::move(*pipeline_builder)); + LOG_INFO(logger, "build pipeline {} ms", stopwatch.elapsedMicroseconds() / 1000.0); + + LOG_DEBUG( + logger, "clickhouse plan [optimization={}]:\n{}", settings.query_plan_enable_optimizations, PlanUtil::explainPlan(*query_plan)); + LOG_DEBUG(logger, "clickhouse pipeline:\n{}", QueryPipelineUtil::explainPipeline(pipeline)); + + return std::make_unique( + context, std::move(query_plan), std::move(pipeline), query_plan->getCurrentDataStream().header.cloneEmpty()); +} -QueryPlanPtr SerializedPlanParser::parse(const std::string & plan) +QueryPlanPtr SerializedPlanParser::parse(const std::string_view & plan) { - auto plan_ptr = std::make_unique(); + substrait::Plan s_plan; /// https://stackoverflow.com/questions/52028583/getting-error-parsing-protobuf-data /// Parsing may fail when the number of recursive layers is large. /// Here, set a limit large enough to avoid this problem. @@ -1769,11 +1765,10 @@ QueryPlanPtr SerializedPlanParser::parse(const std::string & plan) google::protobuf::io::CodedInputStream coded_in(reinterpret_cast(plan.data()), static_cast(plan.size())); coded_in.SetRecursionLimit(100000); - auto ok = plan_ptr->ParseFromCodedStream(&coded_in); - if (!ok) + if (!s_plan.ParseFromCodedStream(&coded_in)) throw Exception(ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from string failed"); - auto res = parse(std::move(plan_ptr)); + auto res = parse(s_plan); #ifndef NDEBUG PlanUtil::checkOuputType(*res); @@ -1788,17 +1783,16 @@ QueryPlanPtr SerializedPlanParser::parse(const std::string & plan) return res; } -QueryPlanPtr SerializedPlanParser::parseJson(const std::string & json_plan) +QueryPlanPtr SerializedPlanParser::parseJson(const std::string_view & json_plan) { - auto plan_ptr = std::make_unique(); - auto s = google::protobuf::util::JsonStringToMessage(absl::string_view(json_plan), plan_ptr.get()); + substrait::Plan plan; + auto s = google::protobuf::util::JsonStringToMessage(json_plan, &plan); if (!s.ok()) throw Exception(ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from json string failed: {}", s.ToString()); - return parse(std::move(plan_ptr)); + return parse(plan); } -SerializedPlanParser::SerializedPlanParser(const ContextPtr & context_) - : context(context_) +SerializedPlanParser::SerializedPlanParser(const ContextPtr & context_) : context(context_) { } @@ -1807,13 +1801,10 @@ ContextMutablePtr SerializedPlanParser::global_context = nullptr; Context::ConfigurationPtr SerializedPlanParser::config = nullptr; void SerializedPlanParser::collectJoinKeys( - const substrait::Expression & condition, - std::vector> & join_keys, - int32_t right_key_start) + const substrait::Expression & condition, std::vector> & join_keys, int32_t right_key_start) { auto condition_name = getFunctionName( - function_mapping.at(std::to_string(condition.scalar_function().function_reference())), - condition.scalar_function()); + function_mapping.at(std::to_string(condition.scalar_function().function_reference())), condition.scalar_function()); if (condition_name == "and") { collectJoinKeys(condition.scalar_function().arguments(0).value(), join_keys, right_key_start); @@ -1863,8 +1854,8 @@ ASTPtr ASTParser::parseToAST(const Names & names, const substrait::Expression & auto substrait_name = function_signature.substr(0, function_signature.find(':')); auto func_parser = FunctionParserFactory::instance().tryGet(substrait_name, plan_parser); - String function_name = func_parser ? func_parser->getName() - : SerializedPlanParser::getFunctionName(function_signature, scalar_function); + String function_name + = func_parser ? func_parser->getName() : SerializedPlanParser::getFunctionName(function_signature, scalar_function); ASTs ast_args; parseFunctionArgumentsToAST(names, scalar_function, ast_args); @@ -1876,9 +1867,7 @@ ASTPtr ASTParser::parseToAST(const Names & names, const substrait::Expression & } void ASTParser::parseFunctionArgumentsToAST( - const Names & names, - const substrait::Expression_ScalarFunction & scalar_function, - ASTs & ast_args) + const Names & names, const substrait::Expression_ScalarFunction & scalar_function, ASTs & ast_args) { const auto & args = scalar_function.arguments(); @@ -2021,12 +2010,12 @@ ASTPtr ASTParser::parseArgumentToAST(const Names & names, const substrait::Expre } } -void SerializedPlanParser::removeNullableForRequiredColumns(const std::set & require_columns, ActionsDAGPtr actions_dag) +void SerializedPlanParser::removeNullableForRequiredColumns( + const std::set & require_columns, const ActionsDAGPtr & actions_dag) const { for (const auto & item : require_columns) { - const auto * require_node = actions_dag->tryFindInOutputs(item); - if (require_node) + if (const auto * require_node = actions_dag->tryFindInOutputs(item)) { auto function_builder = FunctionFactory::instance().get("assumeNotNull", context); ActionsDAG::NodeRawConstPtrs args = {require_node}; @@ -2037,9 +2026,7 @@ void SerializedPlanParser::removeNullableForRequiredColumns(const std::set & columns, - ActionsDAGPtr actions_dag, - std::map & nullable_measure_names) + const std::vector & columns, ActionsDAGPtr actions_dag, std::map & nullable_measure_names) { for (const auto & item : columns) { @@ -2092,86 +2079,23 @@ LocalExecutor::~LocalExecutor() } } - -void LocalExecutor::execute(QueryPlanPtr query_plan) -{ - Stopwatch stopwatch; - - const Settings & settings = context->getSettingsRef(); - current_query_plan = std::move(query_plan); - auto * logger = &Poco::Logger::get("LocalExecutor"); - - QueryPriorities priorities; - auto query_status = std::make_shared( - context, - "", - context->getClientInfo(), - priorities.insert(static_cast(settings.priority)), - CurrentThread::getGroup(), - IAST::QueryKind::Select, - settings, - 0); - - QueryPlanOptimizationSettings optimization_settings{.optimize_plan = settings.query_plan_enable_optimizations}; - auto pipeline_builder = current_query_plan->buildQueryPipeline( - optimization_settings, - BuildQueryPipelineSettings{ - .actions_settings - = ExpressionActionsSettings{.can_compile_expressions = true, .min_count_to_compile_expression = 3, - .compile_expressions = CompileExpressions::yes}, - .process_list_element = query_status}); - - LOG_DEBUG(logger, "clickhouse plan after optimization:\n{}", PlanUtil::explainPlan(*current_query_plan)); - query_pipeline = QueryPipelineBuilder::getPipeline(std::move(*pipeline_builder)); - LOG_DEBUG(logger, "clickhouse pipeline:\n{}", QueryPipelineUtil::explainPipeline(query_pipeline)); - auto t_pipeline = stopwatch.elapsedMicroseconds(); - - executor = std::make_unique(query_pipeline); - auto t_executor = stopwatch.elapsedMicroseconds() - t_pipeline; - stopwatch.stop(); - LOG_INFO( - logger, - "build pipeline {} ms; create executor {} ms;", - t_pipeline / 1000.0, - t_executor / 1000.0); - - header = current_query_plan->getCurrentDataStream().header.cloneEmpty(); - ch_column_to_spark_row = std::make_unique(); -} - -std::unique_ptr LocalExecutor::writeBlockToSparkRow(Block & block) +std::unique_ptr LocalExecutor::writeBlockToSparkRow(const Block & block) const { return ch_column_to_spark_row->convertCHColumnToSparkRow(block); } bool LocalExecutor::hasNext() { - bool has_next; - try + size_t columns = currentBlock().columns(); + if (columns == 0 || isConsumed()) { - size_t columns = currentBlock().columns(); - if (columns == 0 || isConsumed()) - { - auto empty_block = header.cloneEmpty(); - setCurrentBlock(empty_block); - has_next = executor->pull(currentBlock()); - produce(); - } - else - { - has_next = true; - } - } - catch (Exception & e) - { - LOG_ERROR( - &Poco::Logger::get("LocalExecutor"), - "LocalExecutor run query plan failed with message: {}. Plan Explained: \n{}", - e.message(), - PlanUtil::explainPlan(*current_query_plan)); - throw; + auto empty_block = header.cloneEmpty(); + setCurrentBlock(empty_block); + bool has_next = executor->pull(currentBlock()); + produce(); + return has_next; } - return has_next; + return true; } SparkRowInfoPtr LocalExecutor::next() @@ -2246,12 +2170,17 @@ Block & LocalExecutor::getHeader() return header; } -LocalExecutor::LocalExecutor(ContextPtr context_) - : context(context_) +LocalExecutor::LocalExecutor(const ContextPtr & context_, QueryPlanPtr query_plan, QueryPipeline && pipeline, const Block & header_) + : query_pipeline(std::move(pipeline)) + , executor(std::make_unique(query_pipeline)) + , header(header_) + , context(context_) + , ch_column_to_spark_row(std::make_unique()) + , current_query_plan(std::move(query_plan)) { } -std::string LocalExecutor::dumpPipeline() +std::string LocalExecutor::dumpPipeline() const { const auto & processors = query_pipeline.getProcessors(); for (auto & processor : processors) @@ -2275,12 +2204,8 @@ std::string LocalExecutor::dumpPipeline() } NonNullableColumnsResolver::NonNullableColumnsResolver( - const Block & header_, - SerializedPlanParser & parser_, - const substrait::Expression & cond_rel_) - : header(header_) - , parser(parser_) - , cond_rel(cond_rel_) + const Block & header_, SerializedPlanParser & parser_, const substrait::Expression & cond_rel_) + : header(header_), parser(parser_), cond_rel(cond_rel_) { } @@ -2352,8 +2277,7 @@ void NonNullableColumnsResolver::visitNonNullable(const substrait::Expression & } std::string NonNullableColumnsResolver::safeGetFunctionName( - const std::string & function_signature, - const substrait::Expression_ScalarFunction & function) + const std::string & function_signature, const substrait::Expression_ScalarFunction & function) const { try { diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 71cdca58a6ce..82e8c4077841 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -218,6 +218,7 @@ DataTypePtr wrapNullableType(bool nullable, DataTypePtr nested_type); std::string join(const ActionsDAG::NodeRawConstPtrs & v, char c); class SerializedPlanParser; +class LocalExecutor; // Give a condition expression `cond_rel_`, found all columns with nullability that must not containt // null after this filter. @@ -241,7 +242,7 @@ class NonNullableColumnsResolver void visit(const substrait::Expression & expr); void visitNonNullable(const substrait::Expression & expr); - String safeGetFunctionName(const String & function_signature, const substrait::Expression_ScalarFunction & function); + String safeGetFunctionName(const String & function_signature, const substrait::Expression_ScalarFunction & function) const; }; class SerializedPlanParser @@ -257,11 +258,21 @@ class SerializedPlanParser friend class JoinRelParser; friend class MergeTreeRelParser; + std::unique_ptr createExecutor(DB::QueryPlanPtr query_plan); + + DB::QueryPlanPtr parse(const std::string_view & plan); + DB::QueryPlanPtr parse(const substrait::Plan & plan); + public: explicit SerializedPlanParser(const ContextPtr & context); - DB::QueryPlanPtr parse(const std::string & plan); - DB::QueryPlanPtr parseJson(const std::string & json_plan); - DB::QueryPlanPtr parse(std::unique_ptr plan); + + /// UT only + DB::QueryPlanPtr parseJson(const std::string_view & json_plan); + std::unique_ptr createExecutor(const substrait::Plan & plan) { return createExecutor(parse((plan))); } + /// + + template + std::unique_ptr createExecutor(const std::string_view & plan); DB::QueryPlanStepPtr parseReadRealWithLocalFile(const substrait::ReadRel & rel); DB::QueryPlanStepPtr parseReadRealWithJavaIter(const substrait::ReadRel & rel); @@ -372,7 +383,7 @@ class SerializedPlanParser const ActionsDAG::Node * toFunctionNode(ActionsDAGPtr actions_dag, const String & function, const DB::ActionsDAG::NodeRawConstPtrs & args); // remove nullable after isNotNull - void removeNullableForRequiredColumns(const std::set & require_columns, ActionsDAGPtr actions_dag); + void removeNullableForRequiredColumns(const std::set & require_columns, const ActionsDAGPtr & actions_dag) const; std::string getUniqueName(const std::string & name) { return name + "_" + std::to_string(name_no++); } static std::pair parseLiteral(const substrait::Expression_Literal & literal); void wrapNullable( @@ -394,6 +405,12 @@ class SerializedPlanParser const ActionsDAG::Node * addColumn(DB::ActionsDAGPtr actions_dag, const DataTypePtr & type, const Field & field); }; +template +std::unique_ptr SerializedPlanParser::createExecutor(const std::string_view & plan) +{ + return createExecutor(JsonPlan ? parseJson(plan) : parse(plan)); +} + struct SparkBuffer { char * address; @@ -403,16 +420,14 @@ struct SparkBuffer class LocalExecutor : public BlockIterator { public: - LocalExecutor() = default; - explicit LocalExecutor(ContextPtr context); + LocalExecutor(const ContextPtr & context_, QueryPlanPtr query_plan, QueryPipeline && pipeline, const Block & header_); ~LocalExecutor(); - void execute(QueryPlanPtr query_plan); SparkRowInfoPtr next(); Block * nextColumnar(); bool hasNext(); - /// Stop execution and wait for pipeline exit, used when task receives shutdown command or executor receives SIGTERM signal + /// Stop execution, used when task receives shutdown command or executor receives SIGTERM signal void cancel(); Block & getHeader(); @@ -425,13 +440,13 @@ class LocalExecutor : public BlockIterator static void removeExecutor(Int64 handle); private: - std::unique_ptr writeBlockToSparkRow(DB::Block & block); + std::unique_ptr writeBlockToSparkRow(const DB::Block & block) const; void asyncCancel(); void waitCancelFinished(); /// Dump processor runtime information to log - std::string dumpPipeline(); + std::string dumpPipeline() const; QueryPipeline query_pipeline; std::unique_ptr executor; @@ -439,7 +454,7 @@ class LocalExecutor : public BlockIterator ContextPtr context; std::unique_ptr ch_column_to_spark_row; std::unique_ptr spark_buffer; - DB::QueryPlanPtr current_query_plan; + QueryPlanPtr current_query_plan; RelMetricPtr metric; std::vector extra_plan_holder; std::atomic is_cancelled{false}; diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index bbc467879182..9c642d70ec27 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -224,11 +224,9 @@ JNIEXPORT void JNI_OnUnload(JavaVM * vm, void * /*reserved*/) JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeInitNative(JNIEnv * env, jobject, jbyteArray conf_plan) { LOCAL_ENGINE_JNI_METHOD_START - jsize plan_buf_size = env->GetArrayLength(conf_plan); + std::string::size_type plan_buf_size = env->GetArrayLength(conf_plan); jbyte * plan_buf_addr = env->GetByteArrayElements(conf_plan, nullptr); - std::string plan_str; - plan_str.assign(reinterpret_cast(plan_buf_addr), plan_buf_size); - local_engine::BackendInitializerUtil::init(&plan_str); + local_engine::BackendInitializerUtil::init({reinterpret_cast(plan_buf_addr), plan_buf_size}); env->ReleaseByteArrayElements(conf_plan, plan_buf_addr, JNI_ABORT); LOCAL_ENGINE_JNI_METHOD_END(env, ) } @@ -254,11 +252,9 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ auto query_context = local_engine::getAllocator(allocator_id)->query_context; // by task update new configs ( in case of dynamic config update ) - jsize plan_buf_size = env->GetArrayLength(conf_plan); + std::string::size_type plan_buf_size = env->GetArrayLength(conf_plan); jbyte * plan_buf_addr = env->GetByteArrayElements(conf_plan, nullptr); - std::string plan_str; - plan_str.assign(reinterpret_cast(plan_buf_addr), plan_buf_size); - local_engine::BackendInitializerUtil::updateConfig(query_context, &plan_str); + local_engine::BackendInitializerUtil::updateConfig(query_context, {reinterpret_cast(plan_buf_addr), plan_buf_size}); local_engine::SerializedPlanParser parser(query_context); jsize iter_num = env->GetArrayLength(iter_arr); @@ -277,17 +273,14 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ parser.addSplitInfo(std::string{reinterpret_cast(split_info_addr), split_info_size}); } - jsize plan_size = env->GetArrayLength(plan); + std::string::size_type plan_size = env->GetArrayLength(plan); jbyte * plan_address = env->GetByteArrayElements(plan, nullptr); - std::string plan_string; - plan_string.assign(reinterpret_cast(plan_address), plan_size); - auto query_plan = parser.parse(plan_string); - local_engine::LocalExecutor * executor = new local_engine::LocalExecutor(query_context); + local_engine::LocalExecutor * executor + = parser.createExecutor({reinterpret_cast(plan_address), plan_size}).release(); local_engine::LocalExecutor::addExecutor(executor); - LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); + LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); executor->setMetric(parser.getMetric()); executor->setExtraPlanHolder(parser.extra_plan_holder); - executor->execute(std::move(query_plan)); env->ReleaseByteArrayElements(plan, plan_address, JNI_ABORT); env->ReleaseByteArrayElements(conf_plan, plan_buf_addr, JNI_ABORT); return reinterpret_cast(executor); @@ -932,11 +925,10 @@ JNIEXPORT jlong Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniW LOCAL_ENGINE_JNI_METHOD_START auto query_context = local_engine::getAllocator(allocator_id)->query_context; // by task update new configs ( in case of dynamic config update ) - jsize conf_plan_buf_size = env->GetArrayLength(conf_plan); + std::string::size_type conf_plan_buf_size = env->GetArrayLength(conf_plan); jbyte * conf_plan_buf_addr = env->GetByteArrayElements(conf_plan, nullptr); - std::string conf_plan_str; - conf_plan_str.assign(reinterpret_cast(conf_plan_buf_addr), conf_plan_buf_size); - local_engine::BackendInitializerUtil::updateConfig(query_context, &conf_plan_str); + local_engine::BackendInitializerUtil::updateConfig( + query_context, {reinterpret_cast(conf_plan_buf_addr), conf_plan_buf_size}); const auto uuid_str = jstring2string(env, uuid_); const auto task_id = jstring2string(env, task_id_); @@ -1329,14 +1321,11 @@ Java_org_apache_gluten_vectorized_SimpleExpressionEval_createNativeInstance(JNIE local_engine::SerializedPlanParser parser(context); jobject iter = env->NewGlobalRef(input); parser.addInputIter(iter, false); - jsize plan_size = env->GetArrayLength(plan); + std::string::size_type plan_size = env->GetArrayLength(plan); jbyte * plan_address = env->GetByteArrayElements(plan, nullptr); - std::string plan_string; - plan_string.assign(reinterpret_cast(plan_address), plan_size); - auto query_plan = parser.parse(plan_string); - local_engine::LocalExecutor * executor = new local_engine::LocalExecutor(context); + local_engine::LocalExecutor * executor + = parser.createExecutor({reinterpret_cast(plan_address), plan_size}).release(); local_engine::LocalExecutor::addExecutor(executor); - executor->execute(std::move(query_plan)); env->ReleaseByteArrayElements(plan, plan_address, JNI_ABORT); return reinterpret_cast(executor); LOCAL_ENGINE_JNI_METHOD_END(env, -1) diff --git a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp index 89fa4fa961ea..208a3b518d45 100644 --- a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp +++ b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp @@ -154,14 +154,11 @@ DB::ContextMutablePtr global_context; std::move(schema)) .build(); local_engine::SerializedPlanParser parser(global_context); - auto query_plan = parser.parse(std::move(plan)); - local_engine::LocalExecutor local_executor; + auto local_executor = parser.createExecutor(*plan); state.ResumeTiming(); - local_executor.execute(std::move(query_plan)); - while (local_executor.hasNext()) - { - local_engine::SparkRowInfoPtr spark_row_info = local_executor.next(); - } + + while (local_executor->hasNext()) + local_engine::SparkRowInfoPtr spark_row_info = local_executor->next(); } } @@ -212,13 +209,12 @@ DB::ContextMutablePtr global_context; std::move(schema)) .build(); local_engine::SerializedPlanParser parser(SerializedPlanParser::global_context); - auto query_plan = parser.parse(std::move(plan)); - local_engine::LocalExecutor local_executor; + auto local_executor = parser.createExecutor(*plan); state.ResumeTiming(); - local_executor.execute(std::move(query_plan)); - while (local_executor.hasNext()) + + while (local_executor->hasNext()) { - Block * block = local_executor.nextColumnar(); + Block * block = local_executor->nextColumnar(); delete block; } } @@ -238,15 +234,10 @@ DB::ContextMutablePtr global_context; std::ifstream t(path); std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); std::cout << "the plan from: " << path << std::endl; - - auto query_plan = parser.parse(str); - local_engine::LocalExecutor local_executor; + auto local_executor = parser.createExecutor(str); state.ResumeTiming(); - local_executor.execute(std::move(query_plan)); - while (local_executor.hasNext()) - { - [[maybe_unused]] auto * x = local_executor.nextColumnar(); - } + while (local_executor->hasNext()) [[maybe_unused]] + auto * x = local_executor->nextColumnar(); } } @@ -282,14 +273,12 @@ DB::ContextMutablePtr global_context; std::move(schema)) .build(); local_engine::SerializedPlanParser parser(SerializedPlanParser::global_context); - auto query_plan = parser.parse(std::move(plan)); - local_engine::LocalExecutor local_executor; + + auto local_executor = parser.createExecutor(*plan); state.ResumeTiming(); - local_executor.execute(std::move(query_plan)); - while (local_executor.hasNext()) - { - local_engine::SparkRowInfoPtr spark_row_info = local_executor.next(); - } + + while (local_executor->hasNext()) + local_engine::SparkRowInfoPtr spark_row_info = local_executor->next(); } } @@ -320,16 +309,13 @@ DB::ContextMutablePtr global_context; .build(); local_engine::SerializedPlanParser parser(SerializedPlanParser::global_context); - auto query_plan = parser.parse(std::move(plan)); - local_engine::LocalExecutor local_executor; - - local_executor.execute(std::move(query_plan)); + auto local_executor = parser.createExecutor(*plan); local_engine::SparkRowToCHColumn converter; - while (local_executor.hasNext()) + while (local_executor->hasNext()) { - local_engine::SparkRowInfoPtr spark_row_info = local_executor.next(); + local_engine::SparkRowInfoPtr spark_row_info = local_executor->next(); state.ResumeTiming(); - auto block = converter.convertSparkRowInfoToCHColumn(*spark_row_info, local_executor.getHeader()); + auto block = converter.convertSparkRowInfoToCHColumn(*spark_row_info, local_executor->getHeader()); state.PauseTiming(); } state.ResumeTiming(); @@ -368,16 +354,13 @@ DB::ContextMutablePtr global_context; std::move(schema)) .build(); local_engine::SerializedPlanParser parser(SerializedPlanParser::global_context); - auto query_plan = parser.parse(std::move(plan)); - local_engine::LocalExecutor local_executor; - - local_executor.execute(std::move(query_plan)); + auto local_executor = parser.createExecutor(*plan); local_engine::SparkRowToCHColumn converter; - while (local_executor.hasNext()) + while (local_executor->hasNext()) { - local_engine::SparkRowInfoPtr spark_row_info = local_executor.next(); + local_engine::SparkRowInfoPtr spark_row_info = local_executor->next(); state.ResumeTiming(); - auto block = converter.convertSparkRowInfoToCHColumn(*spark_row_info, local_executor.getHeader()); + auto block = converter.convertSparkRowInfoToCHColumn(*spark_row_info, local_executor->getHeader()); state.PauseTiming(); } state.ResumeTiming(); @@ -485,12 +468,8 @@ DB::ContextMutablePtr global_context; y.reserve(cnt); for (auto _ : state) - { for (i = 0; i < cnt; i++) - { y[i] = add(x[i], i); - } - } } [[maybe_unused]] static void BM_TestSumInline(benchmark::State & state) @@ -504,12 +483,8 @@ DB::ContextMutablePtr global_context; y.reserve(cnt); for (auto _ : state) - { for (i = 0; i < cnt; i++) - { y[i] = x[i] + i; - } - } } [[maybe_unused]] static void BM_TestPlus(benchmark::State & state) @@ -545,9 +520,7 @@ DB::ContextMutablePtr global_context; block.insert(y); auto executable_function = function->prepare(arguments); for (auto _ : state) - { auto result = executable_function->execute(block.getColumnsWithTypeAndName(), type, rows, false); - } } [[maybe_unused]] static void BM_TestPlusEmbedded(benchmark::State & state) @@ -847,9 +820,7 @@ QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr right, String left_key, St ASTPtr rkey = std::make_shared(right_key); join->addOnKeys(lkey, rkey, true); for (const auto & column : join->columnsFromJoinedTable()) - { join->addJoinedColumn(column); - } auto left_keys = left->getCurrentDataStream().header.getNamesAndTypesList(); join->addJoinedColumnsAndCorrectTypes(left_keys, true); @@ -920,7 +891,8 @@ BENCHMARK(BM_ParquetRead)->Unit(benchmark::kMillisecond)->Iterations(10); int main(int argc, char ** argv) { - BackendInitializerUtil::init(nullptr); + std::string empty; + BackendInitializerUtil::init(empty); SCOPE_EXIT({ BackendFinalizerUtil::finalizeGlobally(); }); ::benchmark::Initialize(&argc, argv); diff --git a/cpp-ch/local-engine/tests/gluten_test_util.h b/cpp-ch/local-engine/tests/gluten_test_util.h index d4c16e9fbbd8..dba4496d6221 100644 --- a/cpp-ch/local-engine/tests/gluten_test_util.h +++ b/cpp-ch/local-engine/tests/gluten_test_util.h @@ -24,6 +24,7 @@ #include #include #include +#include #include using BlockRowType = DB::ColumnsWithTypeAndName; @@ -60,6 +61,23 @@ AnotherRowType readParquetSchema(const std::string & file); DB::ActionsDAGPtr parseFilter(const std::string & filter, const AnotherRowType & name_and_types); +namespace pb_util +{ +template +std::string JsonStringToBinary(const std::string_view & json) +{ + Message message; + std::string binary; + auto s = google::protobuf::util::JsonStringToMessage(json, &message); + if (!s.ok()) + { + const std::string err_msg{s.message()}; + throw std::runtime_error(err_msg); + } + message.SerializeToString(&binary); + return binary; +} +} } inline DB::DataTypePtr BIGINT() diff --git a/cpp-ch/local-engine/tests/gtest_local_engine.cpp b/cpp-ch/local-engine/tests/gtest_local_engine.cpp index 2d1807841041..962bf9def52e 100644 --- a/cpp-ch/local-engine/tests/gtest_local_engine.cpp +++ b/cpp-ch/local-engine/tests/gtest_local_engine.cpp @@ -16,9 +16,12 @@ */ #include #include +#include +#include + #include -#include #include +#include #include #include #include @@ -28,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -84,13 +86,23 @@ TEST(ReadBufferFromFile, seekBackwards) ASSERT_EQ(x, 8); } +INCBIN(resource_embedded_config_json, SOURCE_DIR "/utils/extern-local-engine/tests/json/gtest_local_engine_config.json"); + +namespace DB +{ +void registerOutputFormatParquet(DB::FormatFactory & factory); +} + int main(int argc, char ** argv) { - auto * init = new String("{\"advancedExtensions\":{\"enhancement\":{\"@type\":\"type.googleapis.com/substrait.Expression\",\"literal\":{\"map\":{\"keyValues\":[{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level\"},\"value\":{\"string\":\"trace\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_before_external_sort\"},\"value\":{\"string\":\"5368709120\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.endpoint\"},\"value\":{\"string\":\"localhost:9000\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.velox.IOThreads\"},\"value\":{\"string\":\"0\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.input_read_timeout\"},\"value\":{\"string\":\"180000\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_settings.query_plan_enable_optimizations\"},\"value\":{\"string\":\"false\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.worker.id\"},\"value\":{\"string\":\"1\"}},{\"key\":{\"string\":\"spark.memory.offHeap.enabled\"},\"value\":{\"string\":\"true\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.iam.role.session.name\"},\"value\":{\"string\":\"\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.input_connect_timeout\"},\"value\":{\"string\":\"180000\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.shuffle.codec\"},\"value\":{\"string\":\"\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.local_engine.settings.log_processors_profiles\"},\"value\":{\"string\":\"true\"}},{\"key\":{\"string\":\"spark.gluten.memory.offHeap.size.in.bytes\"},\"value\":{\"string\":\"10737418240\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.shuffle.codecBackend\"},\"value\":{\"string\":\"\"}},{\"key\":{\"string\":\"spark.sql.orc.compression.codec\"},\"value\":{\"string\":\"snappy\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_before_external_group_by\"},\"value\":{\"string\":\"5368709120\"}},{\"key\":{\"string\":\"spark.hadoop.input.write.timeout\"},\"value\":{\"string\":\"180000\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.secret.key\"},\"value\":{\"string\":\"\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.access.key\"},\"value\":{\"string\":\"\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.dfs_client_log_severity\"},\"value\":{\"string\":\"INFO\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.path.style.access\"},\"value\":{\"string\":\"true\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.timezone\"},\"value\":{\"string\":\"Asia/Shanghai\"}},{\"key\":{\"string\":\"spark.hadoop.input.read.timeout\"},\"value\":{\"string\":\"180000\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.use.instance.credentials\"},\"value\":{\"string\":\"false\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_settings.output_format_orc_compression_method\"},\"value\":{\"string\":\"snappy\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.iam.role\"},\"value\":{\"string\":\"\"}},{\"key\":{\"string\":\"spark.gluten.memory.task.offHeap.size.in.bytes\"},\"value\":{\"string\":\"10737418240\"}},{\"key\":{\"string\":\"spark.hadoop.input.connect.timeout\"},\"value\":{\"string\":\"180000\"}},{\"key\":{\"string\":\"spark.hadoop.dfs.client.log.severity\"},\"value\":{\"string\":\"INFO\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.velox.SplitPreloadPerDriver\"},\"value\":{\"string\":\"2\"}},{\"key\":{\"string\":\"spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.input_write_timeout\"},\"value\":{\"string\":\"180000\"}},{\"key\":{\"string\":\"spark.hadoop.fs.s3a.connection.ssl.enabled\"},\"value\":{\"string\":\"false\"}}]}}}}}"); + BackendInitializerUtil::init(test::pb_util::JsonStringToBinary( + {reinterpret_cast(gresource_embedded_config_jsonData), gresource_embedded_config_jsonSize})); + + auto & factory = FormatFactory::instance(); + DB::registerOutputFormatParquet(factory); - BackendInitializerUtil::init_json(std::move(init)); SCOPE_EXIT({ BackendFinalizerUtil::finalizeGlobally(); }); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); -} +} \ No newline at end of file diff --git a/cpp-ch/local-engine/tests/gtest_parser.cpp b/cpp-ch/local-engine/tests/gtest_parser.cpp index cbe41c90c81a..485740191ea3 100644 --- a/cpp-ch/local-engine/tests/gtest_parser.cpp +++ b/cpp-ch/local-engine/tests/gtest_parser.cpp @@ -14,307 +14,140 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include +#include #include -#include #include + using namespace local_engine; using namespace DB; -std::string splitBinaryFromJson(const std::string & json) +// Plan for https://github.com/ClickHouse/ClickHouse/pull/65234 +INCBIN(resource_embedded_pr_65234_json, SOURCE_DIR "/utils/extern-local-engine/tests/json/clickhouse_pr_65234.json"); + +TEST(SerializedPlanParser, PR65234) { - std::string binary; - substrait::ReadRel::LocalFiles local_files; - auto s = google::protobuf::util::JsonStringToMessage(absl::string_view(json), &local_files); - local_files.SerializeToString(&binary); - return binary; + const std::string split + = R"({"items":[{"uriFile":"file:///home/chang/SourceCode/rebase_gluten/backends-clickhouse/target/scala-2.12/test-classes/tests-working-home/tpch-data/supplier/part-00000-16caa751-9774-470c-bd37-5c84c53373c8-c000.snappy.parquet","length":"84633","parquet":{},"schema":{},"metadataColumns":[{}]}]})"; + SerializedPlanParser parser(SerializedPlanParser::global_context); + parser.addSplitInfo(test::pb_util::JsonStringToBinary(split)); + auto query_plan + = parser.parseJson({reinterpret_cast(gresource_embedded_pr_65234_jsonData), gresource_embedded_pr_65234_jsonSize}); } -std::string JsonPlanFor65234() +#include +#include +#include +#include +#include + +Chunk testChunk() { - // Plan for https://github.com/ClickHouse/ClickHouse/pull/65234 - return R"( + auto nameCol = STRING()->createColumn(); + nameCol->insert("one"); + nameCol->insert("two"); + nameCol->insert("three"); + + auto valueCol = UINT()->createColumn(); + valueCol->insert(1); + valueCol->insert(2); + valueCol->insert(3); + MutableColumns x; + x.push_back(std::move(nameCol)); + x.push_back(std::move(valueCol)); + return {std::move(x), 3}; +} + +TEST(LocalExecutor, StorageObjectStorageSink) { - "extensions": [{ - "extensionFunction": { - "functionAnchor": 1, - "name": "is_not_null:str" - } - }, { - "extensionFunction": { - "functionAnchor": 2, - "name": "equal:str_str" - } - }, { - "extensionFunction": { - "functionAnchor": 3, - "name": "is_not_null:i64" - } - }, { - "extensionFunction": { - "name": "and:bool_bool" - } - }], - "relations": [{ - "root": { - "input": { - "project": { - "common": { - "emit": { - "outputMapping": [2] - } - }, - "input": { - "filter": { - "common": { - "direct": { - } - }, - "input": { - "read": { - "common": { - "direct": { - } - }, - "baseSchema": { - "names": ["r_regionkey", "r_name"], - "struct": { - "types": [{ - "i64": { - "nullability": "NULLABILITY_NULLABLE" - } - }, { - "string": { - "nullability": "NULLABILITY_NULLABLE" - } - }] - }, - "columnTypes": ["NORMAL_COL", "NORMAL_COL"] - }, - "filter": { - "scalarFunction": { - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [{ - "value": { - "scalarFunction": { - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [{ - "value": { - "scalarFunction": { - "functionReference": 1, - "outputType": { - "bool": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - "arguments": [{ - "value": { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - } - } - } - }] - } - } - }, { - "value": { - "scalarFunction": { - "functionReference": 2, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [{ - "value": { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - } - } - } - }, { - "value": { - "literal": { - "string": "EUROPE" - } - } - }] - } - } - }] - } - } - }, { - "value": { - "scalarFunction": { - "functionReference": 3, - "outputType": { - "bool": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - "arguments": [{ - "value": { - "selection": { - "directReference": { - "structField": { - } - } - } - } - }] - } - } - }] - } - }, - "advancedExtension": { - "optimization": { - "@type": "type.googleapis.com/google.protobuf.StringValue", - "value": "isMergeTree\u003d0\n" - } - } - } - }, - "condition": { - "scalarFunction": { - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [{ - "value": { - "scalarFunction": { - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [{ - "value": { - "scalarFunction": { - "functionReference": 1, - "outputType": { - "bool": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - "arguments": [{ - "value": { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - } - } - } - }] - } - } - }, { - "value": { - "scalarFunction": { - "functionReference": 2, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [{ - "value": { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - } - } - } - }, { - "value": { - "literal": { - "string": "EUROPE" - } - } - }] - } - } - }] - } - } - }, { - "value": { - "scalarFunction": { - "functionReference": 3, - "outputType": { - "bool": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - "arguments": [{ - "value": { - "selection": { - "directReference": { - "structField": { - } - } - } - } - }] - } - } - }] - } - } - } - }, - "expressions": [{ - "selection": { - "directReference": { - "structField": { - } - } - } - }] - } - }, - "names": ["r_regionkey#72"], - "outputSchema": { - "types": [{ - "i64": { - "nullability": "NULLABILITY_NULLABLE" - } - }], - "nullability": "NULLABILITY_REQUIRED" - } - } - }] + /// 0. Create ObjectStorage for HDFS + auto settings = SerializedPlanParser::global_context->getSettingsRef(); + const std::string query + = R"(CREATE TABLE hdfs_engine_xxxx (name String, value UInt32) ENGINE=HDFS('hdfs://localhost:8020/clickhouse/test2', 'Parquet'))"; + DB::ParserCreateQuery parser; + std::string error_message; + const char * pos = query.data(); + auto ast = DB::tryParseQuery( + parser, + pos, + pos + query.size(), + error_message, + /* hilite = */ false, + "QUERY TEST", + /* allow_multi_statements = */ false, + 0, + settings.max_parser_depth, + settings.max_parser_backtracks, + true); + auto & create = ast->as(); + auto arg = create.storage->children[0]; + const auto * func = arg->as(); + EXPECT_TRUE(func && func->name == "HDFS"); + + DB::StorageHDFSConfiguration config; + StorageObjectStorage::Configuration::initialize(config, arg->children[0]->children, SerializedPlanParser::global_context, false); + + const std::shared_ptr object_storage + = std::dynamic_pointer_cast(config.createObjectStorage(SerializedPlanParser::global_context, false)); + EXPECT_TRUE(object_storage != nullptr); + + RelativePathsWithMetadata files_with_metadata; + object_storage->listObjects("/clickhouse", files_with_metadata, 0); + + /// 1. Create ObjectStorageSink + DB::StorageObjectStorageSink sink{ + object_storage, config.clone(), {}, {{STRING(), "name"}, {UINT(), "value"}}, SerializedPlanParser::global_context, ""}; + + /// 2. Create Chunk + /// 3. comsume + sink.consume(testChunk()); + sink.onFinish(); } -)"; + +namespace DB +{ +SinkToStoragePtr createFilelinkSink( + const StorageMetadataPtr & metadata_snapshot, + const String & table_name_for_log, + const String & path, + CompressionMethod compression_method, + const std::optional & format_settings, + const String & format_name, + const ContextPtr & context, + int flags); } -TEST(SerializedPlanParser, PR65234) +INCBIN(resource_embedded_readcsv_json, SOURCE_DIR "/utils/extern-local-engine/tests/json/read_student_option_schema.csv.json"); +TEST(LocalExecutor, StorageFileSink) { const std::string split - = R"({"items":[{"uriFile":"file:///part-00000-16caa751-9774-470c-bd37-5c84c53373c8-c000.snappy.parquet","length":"84633","parquet":{},"schema":{},"metadataColumns":[{}]}]}")"; + = R"({"items":[{"uriFile":"file:///home/chang/SourceCode/rebase_gluten/backends-velox/src/test/resources/datasource/csv/student_option_schema.csv","length":"56","text":{"fieldDelimiter":",","maxBlockSize":"8192","header":"1"},"schema":{"names":["id","name","language"],"struct":{"types":[{"string":{"nullability":"NULLABILITY_NULLABLE"}},{"string":{"nullability":"NULLABILITY_NULLABLE"}},{"string":{"nullability":"NULLABILITY_NULLABLE"}}]}},"metadataColumns":[{}]}]})"; SerializedPlanParser parser(SerializedPlanParser::global_context); - parser.addSplitInfo(splitBinaryFromJson(split)); - parser.parseJson(JsonPlanFor65234()); -} + parser.addSplitInfo(test::pb_util::JsonStringToBinary(split)); + auto local_executor = parser.createExecutor( + {reinterpret_cast(gresource_embedded_readcsv_jsonData), gresource_embedded_readcsv_jsonSize}); + + while (local_executor->hasNext()) + { + const Block & x = *local_executor->nextColumnar(); + EXPECT_EQ(4, x.rows()); + } + + StorageInMemoryMetadata metadata; + metadata.setColumns(ColumnsDescription::fromNamesAndTypes({{"name", STRING()}, {"value", UINT()}})); + StorageMetadataPtr metadata_ptr = std::make_shared(metadata); + + auto sink = createFilelinkSink( + metadata_ptr, + "test_table", + "/tmp/test_table.parquet", + CompressionMethod::None, + {}, + "Parquet", + SerializedPlanParser::global_context, + 0); + + sink->consume(testChunk()); + sink->onFinish(); +} \ No newline at end of file diff --git a/cpp-ch/local-engine/tests/json/clickhouse_pr_65234.json b/cpp-ch/local-engine/tests/json/clickhouse_pr_65234.json new file mode 100644 index 000000000000..1c37b68b7144 --- /dev/null +++ b/cpp-ch/local-engine/tests/json/clickhouse_pr_65234.json @@ -0,0 +1,273 @@ +{ + "extensions": [{ + "extensionFunction": { + "functionAnchor": 1, + "name": "is_not_null:str" + } + }, { + "extensionFunction": { + "functionAnchor": 2, + "name": "equal:str_str" + } + }, { + "extensionFunction": { + "functionAnchor": 3, + "name": "is_not_null:i64" + } + }, { + "extensionFunction": { + "name": "and:bool_bool" + } + }], + "relations": [{ + "root": { + "input": { + "project": { + "common": { + "emit": { + "outputMapping": [2] + } + }, + "input": { + "filter": { + "common": { + "direct": { + } + }, + "input": { + "read": { + "common": { + "direct": { + } + }, + "baseSchema": { + "names": ["r_regionkey", "r_name"], + "struct": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }, { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }] + }, + "columnTypes": ["NORMAL_COL", "NORMAL_COL"] + }, + "filter": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }, { + "value": { + "literal": { + "string": "EUROPE" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + } + } + } + }] + } + } + }] + } + }, + "advancedExtension": { + "optimization": { + "@type": "type.googleapis.com/google.protobuf.StringValue", + "value": "isMergeTree\u003d0\n" + } + } + } + }, + "condition": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "scalarFunction": { + "functionReference": 1, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 2, + "outputType": { + "bool": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 1 + } + } + } + } + }, { + "value": { + "literal": { + "string": "EUROPE" + } + } + }] + } + } + }] + } + } + }, { + "value": { + "scalarFunction": { + "functionReference": 3, + "outputType": { + "bool": { + "nullability": "NULLABILITY_REQUIRED" + } + }, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + } + } + } + } + }] + } + } + }] + } + } + } + }, + "expressions": [{ + "selection": { + "directReference": { + "structField": { + } + } + } + }] + } + }, + "names": ["r_regionkey#72"], + "outputSchema": { + "types": [{ + "i64": { + "nullability": "NULLABILITY_NULLABLE" + } + }], + "nullability": "NULLABILITY_REQUIRED" + } + } + }] +} \ No newline at end of file diff --git a/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json b/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json new file mode 100644 index 000000000000..10f0ea3dfdad --- /dev/null +++ b/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json @@ -0,0 +1,269 @@ +{ + "advancedExtensions": { + "enhancement": { + "@type": "type.googleapis.com/substrait.Expression", + "literal": { + "map": { + "keyValues": [ + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.logger.level" + }, + "value": { + "string": "test" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_before_external_sort" + }, + "value": { + "string": "5368709120" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.endpoint" + }, + "value": { + "string": "localhost:9000" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.velox.IOThreads" + }, + "value": { + "string": "0" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.input_read_timeout" + }, + "value": { + "string": "180000" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_settings.query_plan_enable_optimizations" + }, + "value": { + "string": "false" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.worker.id" + }, + "value": { + "string": "1" + } + }, + { + "key": { + "string": "spark.memory.offHeap.enabled" + }, + "value": { + "string": "true" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.iam.role.session.name" + }, + "value": { + "string": "" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.input_connect_timeout" + }, + "value": { + "string": "180000" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.shuffle.codec" + }, + "value": { + "string": "" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.local_engine.settings.log_processors_profiles" + }, + "value": { + "string": "true" + } + }, + { + "key": { + "string": "spark.gluten.memory.offHeap.size.in.bytes" + }, + "value": { + "string": "10737418240" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.shuffle.codecBackend" + }, + "value": { + "string": "" + } + }, + { + "key": { + "string": "spark.sql.orc.compression.codec" + }, + "value": { + "string": "snappy" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_before_external_group_by" + }, + "value": { + "string": "5368709120" + } + }, + { + "key": { + "string": "spark.hadoop.input.write.timeout" + }, + "value": { + "string": "180000" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.secret.key" + }, + "value": { + "string": "" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.access.key" + }, + "value": { + "string": "" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.dfs_client_log_severity" + }, + "value": { + "string": "INFO" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.path.style.access" + }, + "value": { + "string": "true" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.timezone" + }, + "value": { + "string": "Asia/Shanghai" + } + }, + { + "key": { + "string": "spark.hadoop.input.read.timeout" + }, + "value": { + "string": "180000" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.use.instance.credentials" + }, + "value": { + "string": "false" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_settings.output_format_orc_compression_method" + }, + "value": { + "string": "snappy" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.iam.role" + }, + "value": { + "string": "" + } + }, + { + "key": { + "string": "spark.gluten.memory.task.offHeap.size.in.bytes" + }, + "value": { + "string": "10737418240" + } + }, + { + "key": { + "string": "spark.hadoop.input.connect.timeout" + }, + "value": { + "string": "180000" + } + }, + { + "key": { + "string": "spark.hadoop.dfs.client.log.severity" + }, + "value": { + "string": "INFO" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.velox.SplitPreloadPerDriver" + }, + "value": { + "string": "2" + } + }, + { + "key": { + "string": "spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.input_write_timeout" + }, + "value": { + "string": "180000" + } + }, + { + "key": { + "string": "spark.hadoop.fs.s3a.connection.ssl.enabled" + }, + "value": { + "string": "false" + } + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/cpp-ch/local-engine/tests/json/read_student_option_schema.csv.json b/cpp-ch/local-engine/tests/json/read_student_option_schema.csv.json new file mode 100644 index 000000000000..f9518d39014a --- /dev/null +++ b/cpp-ch/local-engine/tests/json/read_student_option_schema.csv.json @@ -0,0 +1,77 @@ +{ + "relations": [ + { + "root": { + "input": { + "read": { + "common": { + "direct": {} + }, + "baseSchema": { + "names": [ + "id", + "name", + "language" + ], + "struct": { + "types": [ + { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + } + ] + }, + "columnTypes": [ + "NORMAL_COL", + "NORMAL_COL", + "NORMAL_COL" + ] + }, + "advancedExtension": { + "optimization": { + "@type": "type.googleapis.com/google.protobuf.StringValue", + "value": "isMergeTree=0\n" + } + } + } + }, + "names": [ + "id#20", + "name#21", + "language#22" + ], + "outputSchema": { + "types": [ + { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + }, + { + "string": { + "nullability": "NULLABILITY_NULLABLE" + } + } + ], + "nullability": "NULLABILITY_REQUIRED" + } + } + } + ] +} \ No newline at end of file diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 9a37c4a40dd1..3ca5e0313924 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -430,7 +430,9 @@ trait SparkPlanExecApi { * * @return */ - def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] + def genExtendedColumnarPostRules(): List[SparkSession => Rule[SparkPlan]] = { + SparkShimLoader.getSparkShims.getExtendedColumnarPostRules() ::: List() + } def genInjectPostHocResolutionRules(): List[SparkSession => Rule[LogicalPlan]] diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/SubstraitPlanPrinterUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/SubstraitPlanPrinterUtil.scala index 77d5d55f618d..a6ec7cb21fbf 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/SubstraitPlanPrinterUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/SubstraitPlanPrinterUtil.scala @@ -24,37 +24,34 @@ import io.substrait.proto.{NamedStruct, Plan} object SubstraitPlanPrinterUtil extends Logging { - /** Transform Substrait Plan to json format. */ - def substraitPlanToJson(substraintPlan: Plan): String = { + private def typeRegistry( + d: com.google.protobuf.Descriptors.Descriptor): com.google.protobuf.TypeRegistry = { val defaultRegistry = WrappersProto.getDescriptor.getMessageTypes - val registry = com.google.protobuf.TypeRegistry + com.google.protobuf.TypeRegistry .newBuilder() - .add(substraintPlan.getDescriptorForType()) + .add(d) .add(defaultRegistry) .build() - JsonFormat.printer.usingTypeRegistry(registry).print(substraintPlan) + } + private def MessageToJson(message: com.google.protobuf.Message): String = { + val registry = typeRegistry(message.getDescriptorForType) + JsonFormat.printer.usingTypeRegistry(registry).print(message) } - def substraitNamedStructToJson(substraintPlan: NamedStruct): String = { - val defaultRegistry = WrappersProto.getDescriptor.getMessageTypes - val registry = com.google.protobuf.TypeRegistry - .newBuilder() - .add(substraintPlan.getDescriptorForType()) - .add(defaultRegistry) - .build() - JsonFormat.printer.usingTypeRegistry(registry).print(substraintPlan) + /** Transform Substrait Plan to json format. */ + def substraitPlanToJson(substraitPlan: Plan): String = { + MessageToJson(substraitPlan) + } + + def substraitNamedStructToJson(namedStruct: NamedStruct): String = { + MessageToJson(namedStruct) } /** Transform substrait plan json string to PlanNode */ def jsonToSubstraitPlan(planJson: String): Plan = { try { val builder = Plan.newBuilder() - val defaultRegistry = WrappersProto.getDescriptor.getMessageTypes - val registry = com.google.protobuf.TypeRegistry - .newBuilder() - .add(builder.getDescriptorForType) - .add(defaultRegistry) - .build() + val registry = typeRegistry(builder.getDescriptorForType) JsonFormat.parser().usingTypeRegistry(registry).merge(planJson, builder) builder.build() } catch { From 4c52976e4fce98e861da210f13a85a74d45f386e Mon Sep 17 00:00:00 2001 From: Shuai li Date: Tue, 25 Jun 2024 10:28:39 +0800 Subject: [PATCH 327/402] [GLUTEN-6176][CH] Support aggreate avg return decimal (#6177) * Support aggreate avg return decimal * update version * fix rebase * add ut --- .../GlutenClickHouseDecimalSuite.scala | 5 +- .../AggregateFunctionSparkAvg.cpp | 158 ++++++++++++++++++ cpp-ch/local-engine/Common/CHUtil.cpp | 9 +- cpp-ch/local-engine/Common/CHUtil.h | 5 +- .../local-engine/Common/GlutenDecimalUtils.h | 108 ++++++++++++ cpp-ch/local-engine/Parser/RelParser.cpp | 23 ++- .../org/apache/gluten/GlutenConfig.scala | 8 +- 7 files changed, 303 insertions(+), 13 deletions(-) create mode 100644 cpp-ch/local-engine/AggregateFunctions/AggregateFunctionSparkAvg.cpp create mode 100644 cpp-ch/local-engine/Common/GlutenDecimalUtils.h diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala index 088487101081..7320b7c05152 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDecimalSuite.scala @@ -67,9 +67,9 @@ class GlutenClickHouseDecimalSuite private val decimalTPCHTables: Seq[(DecimalType, Seq[Int])] = Seq.apply( (DecimalType.apply(9, 4), Seq()), // 1: ch decimal avg is float - (DecimalType.apply(18, 8), Seq(1)), + (DecimalType.apply(18, 8), Seq()), // 1: ch decimal avg is float, 3/10: all value is null and compare with limit - (DecimalType.apply(38, 19), Seq(1, 3, 10)) + (DecimalType.apply(38, 19), Seq(3, 10)) ) private def createDecimalTables(dataType: DecimalType): Unit = { @@ -337,7 +337,6 @@ class GlutenClickHouseDecimalSuite allowPrecisionLoss => Range .inclusive(1, 22) - .filter(_ != 17) // Ignore Q17 which include avg .foreach { sql_num => { diff --git a/cpp-ch/local-engine/AggregateFunctions/AggregateFunctionSparkAvg.cpp b/cpp-ch/local-engine/AggregateFunctions/AggregateFunctionSparkAvg.cpp new file mode 100644 index 000000000000..5eb3a0b36057 --- /dev/null +++ b/cpp-ch/local-engine/AggregateFunctions/AggregateFunctionSparkAvg.cpp @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace DB +{ +struct Settings; + +namespace ErrorCodes +{ + +} +} + +namespace local_engine +{ +using namespace DB; + + +DataTypePtr getSparkAvgReturnType(const DataTypePtr & arg_type) +{ + const UInt32 precision_value = std::min(getDecimalPrecision(*arg_type) + 4, DecimalUtils::max_precision); + const auto scale_value = std::min(getDecimalScale(*arg_type) + 4, precision_value); + return createDecimal(precision_value, scale_value); +} + +template +requires is_decimal +class AggregateFunctionSparkAvg final : public AggregateFunctionAvg +{ +public: + using Base = AggregateFunctionAvg; + + explicit AggregateFunctionSparkAvg(const DataTypes & argument_types_, UInt32 num_scale_, UInt32 round_scale_) + : Base(argument_types_, createResultType(argument_types_, num_scale_, round_scale_), num_scale_) + , num_scale(num_scale_) + , round_scale(round_scale_) + { + } + + DataTypePtr createResultType(const DataTypes & argument_types_, UInt32 num_scale_, UInt32 round_scale_) + { + const DataTypePtr & data_type = argument_types_[0]; + const UInt32 precision_value = std::min(getDecimalPrecision(*data_type) + 4, DecimalUtils::max_precision); + const auto scale_value = std::min(num_scale_ + 4, precision_value); + return createDecimal(precision_value, scale_value); + } + + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override + { + const DataTypePtr & result_type = this->getResultType(); + auto result_scale = getDecimalScale(*result_type); + WhichDataType which(result_type); + if (which.isDecimal32()) + { + assert_cast &>(to).getData().push_back( + divideDecimalAndUInt(this->data(place), num_scale, result_scale, round_scale)); + } + else if (which.isDecimal64()) + { + assert_cast &>(to).getData().push_back( + divideDecimalAndUInt(this->data(place), num_scale, result_scale, round_scale)); + } + else if (which.isDecimal128()) + { + assert_cast &>(to).getData().push_back( + divideDecimalAndUInt(this->data(place), num_scale, result_scale, round_scale)); + } + else + { + assert_cast &>(to).getData().push_back( + divideDecimalAndUInt(this->data(place), num_scale, result_scale, round_scale)); + } + } + + String getName() const override { return "sparkAvg"; } + +private: + Int128 NO_SANITIZE_UNDEFINED + divideDecimalAndUInt(AvgFraction, UInt64> avg, UInt32 num_scale, UInt32 result_scale, UInt32 round_scale) const + { + auto value = avg.numerator.value; + if (result_scale > num_scale) + { + auto diff = DecimalUtils::scaleMultiplier>(result_scale - num_scale); + value = value * diff; + } + else if (result_scale < num_scale) + { + auto diff = DecimalUtils::scaleMultiplier>(num_scale - result_scale); + value = value / diff; + } + + auto result = value / avg.denominator; + + if (round_scale > result_scale) + return result; + + auto round_diff = DecimalUtils::scaleMultiplier>(result_scale - round_scale); + return (result + round_diff / 2) / round_diff * round_diff; + } + +private: + UInt32 num_scale; + UInt32 round_scale; +}; + +AggregateFunctionPtr +createAggregateFunctionSparkAvg(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings) +{ + assertNoParameters(name, parameters); + assertUnary(name, argument_types); + + AggregateFunctionPtr res; + const DataTypePtr & data_type = argument_types[0]; + if (!isDecimal(data_type)) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}", data_type->getName(), name); + + bool allowPrecisionLoss = settings->get(DECIMAL_OPERATIONS_ALLOW_PREC_LOSS).get(); + const UInt32 p1 = DB::getDecimalPrecision(*data_type); + const UInt32 s1 = DB::getDecimalScale(*data_type); + auto [p2, s2] = GlutenDecimalUtils::LONG_DECIMAL; + auto [_, round_scale] = GlutenDecimalUtils::dividePrecisionScale(p1, s1, p2, s2, allowPrecisionLoss); + + res.reset(createWithDecimalType(*data_type, argument_types, getDecimalScale(*data_type), round_scale)); + return res; +} + +void registerAggregateFunctionSparkAvg(AggregateFunctionFactory & factory) +{ + factory.registerFunction("sparkAvg", createAggregateFunctionSparkAvg); +} + +} diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index ae3f6dbd5208..588cc1cb2599 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -626,6 +626,7 @@ void BackendInitializerUtil::initSettings(std::map & b settings.set("date_time_input_format", "best_effort"); settings.set(MERGETREE_MERGE_AFTER_INSERT, true); settings.set(MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE, false); + settings.set(DECIMAL_OPERATIONS_ALLOW_PREC_LOSS, true); for (const auto & [key, value] : backend_conf_map) { @@ -665,6 +666,11 @@ void BackendInitializerUtil::initSettings(std::map & b settings.set("session_timezone", time_zone_val); LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{} value:{}", "session_timezone", time_zone_val); } + else if (key == DECIMAL_OPERATIONS_ALLOW_PREC_LOSS) + { + settings.set(key, toField(key, value)); + LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{} value:{}", key, value); + } } /// Finally apply some fixed kvs to settings. @@ -788,6 +794,7 @@ void BackendInitializerUtil::updateNewSettings(const DB::ContextMutablePtr & con extern void registerAggregateFunctionCombinatorPartialMerge(AggregateFunctionCombinatorFactory &); extern void registerAggregateFunctionsBloomFilter(AggregateFunctionFactory &); +extern void registerAggregateFunctionSparkAvg(AggregateFunctionFactory &); extern void registerFunctions(FunctionFactory &); void registerAllFunctions() @@ -797,7 +804,7 @@ void registerAllFunctions() DB::registerAggregateFunctions(); auto & agg_factory = AggregateFunctionFactory::instance(); registerAggregateFunctionsBloomFilter(agg_factory); - + registerAggregateFunctionSparkAvg(agg_factory); { /// register aggregate function combinators from local_engine auto & factory = AggregateFunctionCombinatorFactory::instance(); diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 245d7b3d15c4..0321d410a7d5 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -37,7 +37,10 @@ namespace local_engine { static const String MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE = "mergetree.insert_without_local_storage"; static const String MERGETREE_MERGE_AFTER_INSERT = "mergetree.merge_after_insert"; -static const std::unordered_set BOOL_VALUE_SETTINGS{MERGETREE_MERGE_AFTER_INSERT, MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE}; +static const std::string DECIMAL_OPERATIONS_ALLOW_PREC_LOSS = "spark.sql.decimalOperations.allowPrecisionLoss"; + +static const std::unordered_set BOOL_VALUE_SETTINGS{ + MERGETREE_MERGE_AFTER_INSERT, MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE, DECIMAL_OPERATIONS_ALLOW_PREC_LOSS}; static const std::unordered_set LONG_VALUE_SETTINGS{ "optimize.maxfilesize", "optimize.minFileSize", "mergetree.max_num_part_per_merge_task"}; diff --git a/cpp-ch/local-engine/Common/GlutenDecimalUtils.h b/cpp-ch/local-engine/Common/GlutenDecimalUtils.h new file mode 100644 index 000000000000..32af66ec590e --- /dev/null +++ b/cpp-ch/local-engine/Common/GlutenDecimalUtils.h @@ -0,0 +1,108 @@ +/* +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + + +namespace local_engine +{ + +class GlutenDecimalUtils +{ +public: + static constexpr size_t MAX_PRECISION = 38; + static constexpr size_t MAX_SCALE = 38; + static constexpr auto system_Default = std::tuple(MAX_PRECISION, 18); + static constexpr auto user_Default = std::tuple(10, 0); + static constexpr size_t MINIMUM_ADJUSTED_SCALE = 6; + + // The decimal types compatible with other numeric types + static constexpr auto BOOLEAN_DECIMAL = std::tuple(1, 0); + static constexpr auto BYTE_DECIMAL = std::tuple(3, 0); + static constexpr auto SHORT_DECIMAL = std::tuple(5, 0); + static constexpr auto INT_DECIMAL = std::tuple(10, 0); + static constexpr auto LONG_DECIMAL = std::tuple(20, 0); + static constexpr auto FLOAT_DECIMAL = std::tuple(14, 7); + static constexpr auto DOUBLE_DECIMAL = std::tuple(30, 15); + static constexpr auto BIGINT_DECIMAL = std::tuple(MAX_PRECISION, 0); + + static std::tuple adjustPrecisionScale(size_t precision, size_t scale) + { + if (precision <= MAX_PRECISION) + { + // Adjustment only needed when we exceed max precision + return std::tuple(precision, scale); + } + else if (scale < 0) + { + // Decimal can have negative scale (SPARK-24468). In this case, we cannot allow a precision + // loss since we would cause a loss of digits in the integer part. + // In this case, we are likely to meet an overflow. + return std::tuple(GlutenDecimalUtils::MAX_PRECISION, scale); + } + else + { + // Precision/scale exceed maximum precision. Result must be adjusted to MAX_PRECISION. + auto intDigits = precision - scale; + // If original scale is less than MINIMUM_ADJUSTED_SCALE, use original scale value; otherwise + // preserve at least MINIMUM_ADJUSTED_SCALE fractional digits + auto minScaleValue = std::min(scale, GlutenDecimalUtils::MINIMUM_ADJUSTED_SCALE); + // The resulting scale is the maximum between what is available without causing a loss of + // digits for the integer part of the decimal and the minimum guaranteed scale, which is + // computed above + auto adjustedScale = std::max(GlutenDecimalUtils::MAX_PRECISION - intDigits, minScaleValue); + + return std::tuple(GlutenDecimalUtils::MAX_PRECISION, adjustedScale); + } + } + + static std::tuple dividePrecisionScale(size_t p1, size_t s1, size_t p2, size_t s2, bool allowPrecisionLoss) + { + if (allowPrecisionLoss) + { + // Precision: p1 - s1 + s2 + max(6, s1 + p2 + 1) + // Scale: max(6, s1 + p2 + 1) + const size_t intDig = p1 - s1 + s2; + const size_t scale = std::max(MINIMUM_ADJUSTED_SCALE, s1 + p2 + 1); + const size_t precision = intDig + scale; + return adjustPrecisionScale(precision, scale); + } + else + { + auto intDig = std::min(MAX_SCALE, p1 - s1 + s2); + auto decDig = std::min(MAX_SCALE, std::max(static_cast(6), s1 + p2 + 1)); + auto diff = (intDig + decDig) - MAX_SCALE; + if (diff > 0) + { + decDig -= diff / 2 + 1; + intDig = MAX_SCALE - decDig; + } + return std::tuple(intDig + decDig, decDig); + } + } + + static std::tuple widerDecimalType(const size_t p1, const size_t s1, const size_t p2, const size_t s2) + { + // max(s1, s2) + max(p1-s1, p2-s2), max(s1, s2) + auto scale = std::max(s1, s2); + auto range = std::max(p1 - s1, p2 - s2); + return std::tuple(range + scale, scale); + } + +}; + +} diff --git a/cpp-ch/local-engine/Parser/RelParser.cpp b/cpp-ch/local-engine/Parser/RelParser.cpp index 7fc807827109..282339c4d641 100644 --- a/cpp-ch/local-engine/Parser/RelParser.cpp +++ b/cpp-ch/local-engine/Parser/RelParser.cpp @@ -15,12 +15,16 @@ * limitations under the License. */ #include "RelParser.h" + #include +#include + #include +#include #include -#include -#include #include +#include + namespace DB { @@ -38,7 +42,20 @@ AggregateFunctionPtr RelParser::getAggregateFunction( { auto & factory = AggregateFunctionFactory::instance(); auto action = NullsAction::EMPTY; - return factory.get(name, action, arg_types, parameters, properties); + + String function_name = name; + if (name == "avg" && isDecimal(removeNullable(arg_types[0]))) + function_name = "sparkAvg"; + else if (name == "avgPartialMerge") + { + if (auto agg_func = typeid_cast(arg_types[0].get()); + !agg_func->getArgumentsDataTypes().empty() && isDecimal(removeNullable(agg_func->getArgumentsDataTypes()[0]))) + { + function_name = "sparkAvgPartialMerge"; + } + } + + return factory.get(function_name, action, arg_types, parameters, properties); } std::optional RelParser::parseSignatureFunctionName(UInt32 function_ref) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 148e8cdc067c..4b4e29e7d0fb 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -718,7 +718,9 @@ object GlutenConfig { GLUTEN_OFFHEAP_SIZE_IN_BYTES_KEY, GLUTEN_TASK_OFFHEAP_SIZE_IN_BYTES_KEY, - GLUTEN_OFFHEAP_ENABLED + GLUTEN_OFFHEAP_ENABLED, + SESSION_LOCAL_TIMEZONE.key, + DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key ) nativeConfMap.putAll(conf.filter(e => keys.contains(e._1)).asJava) @@ -735,10 +737,6 @@ object GlutenConfig { .filter(_._1.startsWith(SPARK_ABFS_ACCOUNT_KEY)) .foreach(entry => nativeConfMap.put(entry._1, entry._2)) - conf - .filter(_._1.startsWith(SQLConf.SESSION_LOCAL_TIMEZONE.key)) - .foreach(entry => nativeConfMap.put(entry._1, entry._2)) - // return nativeConfMap } From cf04f0fe3e338169492a28a35cb3562d5d29cdaa Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Tue, 25 Jun 2024 10:34:50 +0800 Subject: [PATCH 328/402] [GLUTEN-5659][VL] Add more configs for AWS s3 (#5660) Add more configs for AWS s3 spark.gluten.velox.fs.s3a.retry.mode spark.gluten.velox.fs.s3a.connect.timeout spark.hadoop.fs.s3a.retry.limit spark.hadoop.fs.s3a.connection.maximum --- cpp/velox/utils/ConfigExtractor.cc | 23 ++++++++++++++ docs/Configuration.md | 2 ++ .../org/apache/gluten/GlutenConfig.scala | 30 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index a71f143225b9..816166351c0e 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -34,6 +34,13 @@ const bool kVeloxFileHandleCacheEnabledDefault = false; // Log granularity of AWS C++ SDK const std::string kVeloxAwsSdkLogLevel = "spark.gluten.velox.awsSdkLogLevel"; const std::string kVeloxAwsSdkLogLevelDefault = "FATAL"; +// Retry mode for AWS s3 +const std::string kVeloxS3RetryMode = "spark.gluten.velox.fs.s3a.retry.mode"; +const std::string kVeloxS3RetryModeDefault = "legacy"; +// Connection timeout for AWS s3 +const std::string kVeloxS3ConnectTimeout = "spark.gluten.velox.fs.s3a.connect.timeout"; +// Using default fs.s3a.connection.timeout value in hadoop +const std::string kVeloxS3ConnectTimeoutDefault = "200s"; } // namespace namespace gluten { @@ -64,6 +71,10 @@ std::shared_ptr getHiveConfig(std::shared_ptr< bool useInstanceCredentials = conf->get("spark.hadoop.fs.s3a.use.instance.credentials", false); std::string iamRole = conf->get("spark.hadoop.fs.s3a.iam.role", ""); std::string iamRoleSessionName = conf->get("spark.hadoop.fs.s3a.iam.role.session.name", ""); + std::string retryMaxAttempts = conf->get("spark.hadoop.fs.s3a.retry.limit", "20"); + std::string retryMode = conf->get(kVeloxS3RetryMode, kVeloxS3RetryModeDefault); + std::string maxConnections = conf->get("spark.hadoop.fs.s3a.connection.maximum", "15"); + std::string connectTimeout = conf->get(kVeloxS3ConnectTimeout, kVeloxS3ConnectTimeoutDefault); std::string awsSdkLogLevel = conf->get(kVeloxAwsSdkLogLevel, kVeloxAwsSdkLogLevelDefault); @@ -79,6 +90,14 @@ std::shared_ptr getHiveConfig(std::shared_ptr< if (envAwsEndpoint != nullptr) { awsEndpoint = std::string(envAwsEndpoint); } + const char* envRetryMaxAttempts = std::getenv("AWS_MAX_ATTEMPTS"); + if (envRetryMaxAttempts != nullptr) { + retryMaxAttempts = std::string(envRetryMaxAttempts); + } + const char* envRetryMode = std::getenv("AWS_RETRY_MODE"); + if (envRetryMode != nullptr) { + retryMode = std::string(envRetryMode); + } if (useInstanceCredentials) { hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3UseInstanceCredentials] = "true"; @@ -98,6 +117,10 @@ std::shared_ptr getHiveConfig(std::shared_ptr< hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3SSLEnabled] = sslEnabled ? "true" : "false"; hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3PathStyleAccess] = pathStyleAccess ? "true" : "false"; hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3LogLevel] = awsSdkLogLevel; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3MaxAttempts] = retryMaxAttempts; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3RetryMode] = retryMode; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3MaxConnections] = maxConnections; + hiveConfMap[facebook::velox::connector::hive::HiveConfig::kS3ConnectTimeout] = connectTimeout; #endif #ifdef ENABLE_GCS diff --git a/docs/Configuration.md b/docs/Configuration.md index 089675286f68..2c2bd4de11f2 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -89,6 +89,8 @@ The following configurations are related to Velox settings. | spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes | Set the max coalesced bytes for velox file scan. | | | spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct | Set prefetch cache min pct for velox file scan. | | | spark.gluten.velox.awsSdkLogLevel | Log granularity of AWS C++ SDK in velox. | FATAL | +| spark.gluten.velox.fs.s3a.retry.mode | Retry mode for AWS s3 connection error, can be "legacy", "standard" and "adaptive". | legacy | +| spark.gluten.velox.fs.s3a.connect.timeout | Timeout for AWS s3 connection. | 1s | | spark.gluten.sql.columnar.backend.velox.orc.scan.enabled | Enable velox orc scan. If disabled, vanilla spark orc scan will be used. | true | | spark.gluten.sql.complexType.scan.fallback.enabled | Force fallback for complex type scan, including struct, map, array. | true | diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 4b4e29e7d0fb..cc2d6ac5fdef 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -436,6 +436,10 @@ class GlutenConfig(conf: SQLConf) extends Logging { def awsSdkLogLevel: String = conf.getConf(AWS_SDK_LOG_LEVEL) + def awsS3RetryMode: String = conf.getConf(AWS_S3_RETRY_MODE) + + def awsConnectionTimeout: String = conf.getConf(AWS_S3_CONNECT_TIMEOUT) + def enableCastAvgAggregateFunction: Boolean = conf.getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED) def enableGlutenCostEvaluator: Boolean = conf.getConf(COST_EVALUATOR_ENABLED) @@ -488,6 +492,10 @@ object GlutenConfig { val SPARK_S3_IAM: String = HADOOP_PREFIX + S3_IAM_ROLE val S3_IAM_ROLE_SESSION_NAME = "fs.s3a.iam.role.session.name" val SPARK_S3_IAM_SESSION_NAME: String = HADOOP_PREFIX + S3_IAM_ROLE_SESSION_NAME + val S3_RETRY_MAX_ATTEMPTS = "fs.s3a.retry.limit" + val SPARK_S3_RETRY_MAX_ATTEMPTS: String = HADOOP_PREFIX + S3_RETRY_MAX_ATTEMPTS + val S3_CONNECTION_MAXIMUM = "fs.s3a.connection.maximum" + val SPARK_S3_CONNECTION_MAXIMUM: String = HADOOP_PREFIX + S3_CONNECTION_MAXIMUM // Hardware acceleraters backend val GLUTEN_SHUFFLE_CODEC_BACKEND = "spark.gluten.sql.columnar.shuffle.codecBackend" @@ -642,6 +650,10 @@ object GlutenConfig { SPARK_S3_USE_INSTANCE_CREDENTIALS, SPARK_S3_IAM, SPARK_S3_IAM_SESSION_NAME, + SPARK_S3_RETRY_MAX_ATTEMPTS, + SPARK_S3_CONNECTION_MAXIMUM, + AWS_S3_CONNECT_TIMEOUT.key, + AWS_S3_RETRY_MODE.key, AWS_SDK_LOG_LEVEL.key, // gcs config SPARK_GCS_STORAGE_ROOT_URL, @@ -693,6 +705,10 @@ object GlutenConfig { (SPARK_S3_USE_INSTANCE_CREDENTIALS, "false"), (SPARK_S3_IAM, ""), (SPARK_S3_IAM_SESSION_NAME, ""), + (SPARK_S3_RETRY_MAX_ATTEMPTS, "20"), + (SPARK_S3_CONNECTION_MAXIMUM, "15"), + (AWS_S3_CONNECT_TIMEOUT.key, AWS_S3_CONNECT_TIMEOUT.defaultValueString), + (AWS_S3_RETRY_MODE.key, AWS_S3_RETRY_MODE.defaultValueString), ( COLUMNAR_VELOX_CONNECTOR_IO_THREADS.key, conf.getOrElse(GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY, "-1")), @@ -1941,6 +1957,20 @@ object GlutenConfig { .stringConf .createWithDefault("FATAL") + val AWS_S3_RETRY_MODE = + buildConf("spark.gluten.velox.fs.s3a.retry.mode") + .internal() + .doc("Retry mode for AWS s3 connection error: legacy, standard and adaptive.") + .stringConf + .createWithDefault("legacy") + + val AWS_S3_CONNECT_TIMEOUT = + buildConf("spark.gluten.velox.fs.s3a.connect.timeout") + .internal() + .doc("Timeout for AWS s3 connection.") + .stringConf + .createWithDefault("200s") + val VELOX_ORC_SCAN_ENABLED = buildStaticConf("spark.gluten.sql.columnar.backend.velox.orc.scan.enabled") .internal() From de26ed2dad41d2d1e893c8d1b3ae806385d9972f Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Tue, 25 Jun 2024 16:10:05 +0800 Subject: [PATCH 329/402] [CH] Support flatten (#6194) [CH] Support flatten Co-authored-by: liuneng1994 --- .../gluten/utils/CHExpressionUtil.scala | 1 - cpp-ch/clickhouse.version | 3 +- .../Functions/SparkArrayFlatten.cpp | 160 ++++++++++++++++++ .../Parser/SerializedPlanParser.h | 1 + .../clickhouse/ClickHouseTestSettings.scala | 2 +- .../sql/GlutenDataFrameFunctionsSuite.scala | 82 +++++++++ .../clickhouse/ClickHouseTestSettings.scala | 2 +- .../sql/GlutenDataFrameFunctionsSuite.scala | 82 +++++++++ 8 files changed, 329 insertions(+), 4 deletions(-) create mode 100644 cpp-ch/local-engine/Functions/SparkArrayFlatten.cpp diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index cf45c1118f13..e9bee84396f8 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -209,7 +209,6 @@ object CHExpressionUtil { UNIX_MICROS -> DefaultValidator(), TIMESTAMP_MILLIS -> DefaultValidator(), TIMESTAMP_MICROS -> DefaultValidator(), - FLATTEN -> DefaultValidator(), STACK -> DefaultValidator() ) } diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 4a3088e54309..54d0a74c5bb4 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,4 @@ CH_ORG=Kyligence CH_BRANCH=rebase_ch/20240621 -CH_COMMIT=acf666c1c4f +CH_COMMIT=c811cbb985f + diff --git a/cpp-ch/local-engine/Functions/SparkArrayFlatten.cpp b/cpp-ch/local-engine/Functions/SparkArrayFlatten.cpp new file mode 100644 index 000000000000..d39bca5ea104 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkArrayFlatten.cpp @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + +/// arrayFlatten([[1, 2, 3], [4, 5]]) = [1, 2, 3, 4, 5] - flatten array. +class SparkArrayFlatten : public IFunction +{ +public: + static constexpr auto name = "sparkArrayFlatten"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!isArray(arguments[0])) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}, expected Array", + arguments[0]->getName(), getName()); + + DataTypePtr nested_type = arguments[0]; + nested_type = checkAndGetDataType(removeNullable(nested_type).get())->getNestedType(); + return nested_type; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + /** We create an array column with array elements as the most deep elements of nested arrays, + * and construct offsets by selecting elements of most deep offsets by values of ancestor offsets. + * +Example 1: + +Source column: Array(Array(UInt8)): +Row 1: [[1, 2, 3], [4, 5]], Row 2: [[6], [7, 8]] +data: [1, 2, 3], [4, 5], [6], [7, 8] +offsets: 2, 4 +data.data: 1 2 3 4 5 6 7 8 +data.offsets: 3 5 6 8 + +Result column: Array(UInt8): +Row 1: [1, 2, 3, 4, 5], Row 2: [6, 7, 8] +data: 1 2 3 4 5 6 7 8 +offsets: 5 8 + +Result offsets are selected from the most deep (data.offsets) by previous deep (offsets) (and values are decremented by one): +3 5 6 8 + ^ ^ + +Example 2: + +Source column: Array(Array(Array(UInt8))): +Row 1: [[], [[1], [], [2, 3]]], Row 2: [[[4]]] + +most deep data: 1 2 3 4 + +offsets1: 2 3 +offsets2: 0 3 4 +- ^ ^ - select by prev offsets +offsets3: 1 1 3 4 +- ^ ^ - select by prev offsets + +result offsets: 3, 4 +result: Row 1: [1, 2, 3], Row2: [4] + */ + + const ColumnArray * src_col = checkAndGetColumn(arguments[0].column.get()); + + if (!src_col) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} in argument of function 'arrayFlatten'", + arguments[0].column->getName()); + + const IColumn::Offsets & src_offsets = src_col->getOffsets(); + + ColumnArray::ColumnOffsets::MutablePtr result_offsets_column; + const IColumn::Offsets * prev_offsets = &src_offsets; + const IColumn * prev_data = &src_col->getData(); + bool nullable = prev_data->isNullable(); + // when array has null element, return null + if (nullable) + { + const ColumnNullable * nullable_column = checkAndGetColumn(prev_data); + prev_data = nullable_column->getNestedColumnPtr().get(); + for (size_t i = 0; i < nullable_column->size(); i++) + { + if (nullable_column->isNullAt(i)) + { + auto res= nullable_column->cloneEmpty(); + res->insertManyDefaults(input_rows_count); + return res; + } + } + } + if (isNothing(prev_data->getDataType())) + return prev_data->cloneResized(input_rows_count); + // only flatten one dimension + if (const ColumnArray * next_col = checkAndGetColumn(prev_data)) + { + result_offsets_column = ColumnArray::ColumnOffsets::create(input_rows_count); + + IColumn::Offsets & result_offsets = result_offsets_column->getData(); + + const IColumn::Offsets * next_offsets = &next_col->getOffsets(); + + for (size_t i = 0; i < input_rows_count; ++i) + result_offsets[i] = (*next_offsets)[(*prev_offsets)[i] - 1]; /// -1 array subscript is Ok, see PaddedPODArray + prev_data = &next_col->getData(); + } + + auto res = ColumnArray::create( + prev_data->getPtr(), + result_offsets_column ? std::move(result_offsets_column) : src_col->getOffsetsPtr()); + if (nullable) + return makeNullable(res); + return res; + } + +private: + String getName() const override + { + return name; + } +}; + +REGISTER_FUNCTION(SparkArrayFlatten) +{ + factory.registerFunction(); +} + +} diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 82e8c4077841..aa18197e5647 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -180,6 +180,7 @@ static const std::map SCALAR_FUNCTIONS {"array", "array"}, {"shuffle", "arrayShuffle"}, {"range", "range"}, /// dummy mapping + {"flatten", "sparkArrayFlatten"}, // map functions {"map", "map"}, diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 8572ef54d5c8..1626716805cb 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -172,6 +172,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("shuffle function - array for primitive type not containing null") .exclude("shuffle function - array for primitive type containing null") .exclude("shuffle function - array for non-primitive type") + .exclude("flatten function") enableSuite[GlutenDataFrameHintSuite] enableSuite[GlutenDataFrameImplicitsSuite] enableSuite[GlutenDataFrameJoinSuite].exclude( @@ -674,7 +675,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("Sequence with default step") .exclude("Reverse") .exclude("elementAt") - .exclude("Flatten") .exclude("ArrayRepeat") .exclude("Array remove") .exclude("Array Distinct") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala index 2b0b40790a76..e64f760ab55f 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala @@ -49,4 +49,86 @@ class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenS false ) } + + testGluten("flatten function") { + // Test cases with a primitive type + val intDF = Seq( + (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))), + (Seq(Seq(1, 2))), + (Seq(Seq(1), Seq.empty)), + (Seq(Seq.empty, Seq(1))) + ).toDF("i") + + val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)), Row(Seq(1)), Row(Seq(1))) + + def testInt(): Unit = { + checkAnswer(intDF.select(flatten($"i")), intDFResult) + checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult) + } + + // Test with local relation, the Project will be evaluated without codegen + testInt() + // Test with cached relation, the Project will be evaluated with codegen + intDF.cache() + testInt() + + // Test cases with non-primitive types + val strDF = Seq( + (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))), + (Seq(Seq("a", "b"))), + (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))), + (Seq(Seq("a"), Seq.empty)), + (Seq(Seq.empty, Seq("a"))) + ).toDF("s") + + val strDFResult = Seq( + Row(Seq("a", "b", "c", "d", "e", "f")), + Row(Seq("a", "b")), + Row(Seq("a", null, null, "b", null, null)), + Row(Seq("a")), + Row(Seq("a"))) + + def testString(): Unit = { + checkAnswer(strDF.select(flatten($"s")), strDFResult) + checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult) + } + + // Test with local relation, the Project will be evaluated without codegen + testString() + // Test with cached relation, the Project will be evaluated with codegen + strDF.cache() + testString() + + val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr") + + def testArray(): Unit = { + checkAnswer( + arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6, null)))"), + Seq(Row(Seq(1, 2, 3, null, 5, 6, null)))) + checkAnswer( + arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"), + Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3))))) + } + + // Test with local relation, the Project will be evaluated without codegen + testArray() + // Test with cached relation, the Project will be evaluated with codegen + arrDF.cache() + testArray() + + // Error test cases + val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr") + intercept[AnalysisException] { + oneRowDF.select(flatten($"arr")) + } + intercept[AnalysisException] { + oneRowDF.select(flatten($"i")) + } + intercept[AnalysisException] { + oneRowDF.select(flatten($"s")) + } + intercept[AnalysisException] { + oneRowDF.selectExpr("flatten(null)") + } + } } diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 50e7929e4619..3147c7c3dbf3 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -190,6 +190,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("shuffle function - array for primitive type not containing null") .exclude("shuffle function - array for primitive type containing null") .exclude("shuffle function - array for non-primitive type") + .exclude("flatten function") enableSuite[GlutenDataFrameHintSuite] enableSuite[GlutenDataFrameImplicitsSuite] enableSuite[GlutenDataFrameJoinSuite].exclude( @@ -714,7 +715,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("Sequence with default step") .exclude("Reverse") .exclude("elementAt") - .exclude("Flatten") .exclude("ArrayRepeat") .exclude("Array remove") .exclude("Array Distinct") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala index 2b0b40790a76..e64f760ab55f 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameFunctionsSuite.scala @@ -49,4 +49,86 @@ class GlutenDataFrameFunctionsSuite extends DataFrameFunctionsSuite with GlutenS false ) } + + testGluten("flatten function") { + // Test cases with a primitive type + val intDF = Seq( + (Seq(Seq(1, 2, 3), Seq(4, 5), Seq(6))), + (Seq(Seq(1, 2))), + (Seq(Seq(1), Seq.empty)), + (Seq(Seq.empty, Seq(1))) + ).toDF("i") + + val intDFResult = Seq(Row(Seq(1, 2, 3, 4, 5, 6)), Row(Seq(1, 2)), Row(Seq(1)), Row(Seq(1))) + + def testInt(): Unit = { + checkAnswer(intDF.select(flatten($"i")), intDFResult) + checkAnswer(intDF.selectExpr("flatten(i)"), intDFResult) + } + + // Test with local relation, the Project will be evaluated without codegen + testInt() + // Test with cached relation, the Project will be evaluated with codegen + intDF.cache() + testInt() + + // Test cases with non-primitive types + val strDF = Seq( + (Seq(Seq("a", "b"), Seq("c"), Seq("d", "e", "f"))), + (Seq(Seq("a", "b"))), + (Seq(Seq("a", null), Seq(null, "b"), Seq(null, null))), + (Seq(Seq("a"), Seq.empty)), + (Seq(Seq.empty, Seq("a"))) + ).toDF("s") + + val strDFResult = Seq( + Row(Seq("a", "b", "c", "d", "e", "f")), + Row(Seq("a", "b")), + Row(Seq("a", null, null, "b", null, null)), + Row(Seq("a")), + Row(Seq("a"))) + + def testString(): Unit = { + checkAnswer(strDF.select(flatten($"s")), strDFResult) + checkAnswer(strDF.selectExpr("flatten(s)"), strDFResult) + } + + // Test with local relation, the Project will be evaluated without codegen + testString() + // Test with cached relation, the Project will be evaluated with codegen + strDF.cache() + testString() + + val arrDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr") + + def testArray(): Unit = { + checkAnswer( + arrDF.selectExpr("flatten(array(arr, array(null, 5), array(6, null)))"), + Seq(Row(Seq(1, 2, 3, null, 5, 6, null)))) + checkAnswer( + arrDF.selectExpr("flatten(array(array(arr, arr), array(arr)))"), + Seq(Row(Seq(Seq(1, 2, 3), Seq(1, 2, 3), Seq(1, 2, 3))))) + } + + // Test with local relation, the Project will be evaluated without codegen + testArray() + // Test with cached relation, the Project will be evaluated with codegen + arrDF.cache() + testArray() + + // Error test cases + val oneRowDF = Seq((1, "a", Seq(1, 2, 3))).toDF("i", "s", "arr") + intercept[AnalysisException] { + oneRowDF.select(flatten($"arr")) + } + intercept[AnalysisException] { + oneRowDF.select(flatten($"i")) + } + intercept[AnalysisException] { + oneRowDF.select(flatten($"s")) + } + intercept[AnalysisException] { + oneRowDF.selectExpr("flatten(null)") + } + } } From 3e5b54e64032ab3e860c1636ab4760557a8a1e96 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:15:55 +0800 Subject: [PATCH 330/402] [VL] Fix greatest and least function tests (#6209) [VL] Fix greatest and least function tests. --- .../ScalarFunctionsValidateSuite.scala | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 75b60addfa13..a2baf95ecdc0 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -157,24 +157,28 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { checkLengthAndPlan(df, 1) } - test("greatest function") { - val df = runQueryAndCompare( - "SELECT greatest(l_orderkey, l_orderkey)" + - "from lineitem limit 1")(checkGlutenOperatorMatch[ProjectExecTransformer]) - } - - test("least function") { - val df = runQueryAndCompare( - "SELECT least(l_orderkey, l_orderkey)" + - "from lineitem limit 1")(checkGlutenOperatorMatch[ProjectExecTransformer]) - } - test("Test greatest function") { runQueryAndCompare( "SELECT greatest(l_orderkey, l_orderkey)" + "from lineitem limit 1") { checkGlutenOperatorMatch[ProjectExecTransformer] } + withTempPath { + path => + spark + .sql("""SELECT * + FROM VALUES (CAST(5.345 AS DECIMAL(6, 2)), CAST(5.35 AS DECIMAL(5, 4))), + (CAST(5.315 AS DECIMAL(6, 2)), CAST(5.355 AS DECIMAL(5, 4))), + (CAST(3.345 AS DECIMAL(6, 2)), CAST(4.35 AS DECIMAL(5, 4))) AS data(a, b);""") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("view") + + runQueryAndCompare("SELECT greatest(a, b) from view") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } } test("Test least function") { @@ -183,6 +187,22 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { "from lineitem limit 1") { checkGlutenOperatorMatch[ProjectExecTransformer] } + withTempPath { + path => + spark + .sql("""SELECT * + FROM VALUES (CAST(5.345 AS DECIMAL(6, 2)), CAST(5.35 AS DECIMAL(5, 4))), + (CAST(5.315 AS DECIMAL(6, 2)), CAST(5.355 AS DECIMAL(5, 4))), + (CAST(3.345 AS DECIMAL(6, 2)), CAST(4.35 AS DECIMAL(5, 4))) AS data(a, b);""") + .write + .parquet(path.getCanonicalPath) + + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("view") + + runQueryAndCompare("SELECT least(a, b) from view") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } } test("Test hash function") { From ad0fb0e718bdd7437360e717a4a3fb0ac8fbc6af Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 25 Jun 2024 21:31:42 +0800 Subject: [PATCH 331/402] [VL] Fix udf segfault for static build (#6215) --- cpp/velox/symbols.map | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/velox/symbols.map b/cpp/velox/symbols.map index ebd2b9af0096..525faf3526a1 100644 --- a/cpp/velox/symbols.map +++ b/cpp/velox/symbols.map @@ -6,6 +6,8 @@ }; Java_org_apache_gluten_*; + JNI_OnLoad; + JNI_OnUnload; local: # Hide symbols of static dependencies *; From 22475dacf5122aaedc5e85b38ec496dad7a2a7e2 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 25 Jun 2024 21:43:35 +0800 Subject: [PATCH 332/402] [VL] Daily Update Velox Version (2024_06_25) (#6204) Velox main changes: ``` 1225f773f by joey.ljy, Add session timezone to Parquet PageReader (#9781) 33cdf0a97 by Wei He, Add custom input generator for lead, lag, nth_value, and ntile in WindowFuzzerTest (#8360) 82a12e165 by Deepak Majeti, Remove setup-centos8.sh (#10249) 7be328cac by Zhenyuan Zhao, Make dwrf support taking custom column reader factory (#10267) 1f981ae8f by Orri Erling, Add more size classes (#10139) dc533655f by Masha Basmanova, Add from_unixtime(epoch, hours, minutes) Presto function (#10215) 7f547dbca by Wei He, Add custom result verifiers for min_by and max_by (#9070) 9974a3339 by Wei He, Allow logging input vectors in aggregation fuzzer (#10229) ``` --- .github/workflows/velox_docker.yml | 10 ++++++++++ ep/build-velox/src/get_velox.sh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 5f64c9f7e0e8..31796c15bdd5 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -120,6 +120,12 @@ jobs: with: name: velox-arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Setup tzdata + run: | + if [ "${{ matrix.os }}" = "ubuntu:22.04" ]; then + apt-get update + TZ="Etc/GMT" DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata + fi - name: Setup java and maven run: | if [ "${{ matrix.java }}" = "java-17" ]; then @@ -530,6 +536,10 @@ jobs: with: name: velox-arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ + - name: Setup tzdata + run: | + apt-get update + TZ="Etc/GMT" DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - name: Setup java and maven run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index d3ecddbdfa9a..06998787d45e 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_24 +VELOX_BRANCH=2024_06_25 VELOX_HOME="" #Set on run gluten on HDFS From 524434826b42fd7e5cfda9b6e023efa656e2c6ae Mon Sep 17 00:00:00 2001 From: James Xu Date: Tue, 25 Jun 2024 22:25:04 +0800 Subject: [PATCH 333/402] [GLUTEN-6219] Fix some code style issue for BasicScanExecTransformer.scala (#6220) Co-authored-by: James Xu --- .../execution/BasicScanExecTransformer.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index 3bbd99c50a6a..9d231bbc2891 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.hive.HiveTableScanExecTransformer import org.apache.spark.sql.types.{BooleanType, StringType, StructField, StructType} import com.google.protobuf.StringValue +import io.substrait.proto.NamedStruct import scala.collection.JavaConverters._ @@ -109,19 +110,19 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource } override protected def doTransform(context: SubstraitContext): TransformContext = { - val output = filteRedundantField(outputAttributes()) + val output = filterRedundantField(outputAttributes()) val typeNodes = ConverterUtils.collectAttributeTypeNodes(output) val nameList = ConverterUtils.collectAttributeNamesWithoutExprId(output) val columnTypeNodes = output.map { attr => if (getPartitionSchema.exists(_.name.equals(attr.name))) { - new ColumnTypeNode(1) + new ColumnTypeNode(NamedStruct.ColumnType.PARTITION_COL_VALUE) } else if (SparkShimLoader.getSparkShims.isRowIndexMetadataColumn(attr.name)) { - new ColumnTypeNode(3) + new ColumnTypeNode(NamedStruct.ColumnType.ROWINDEX_COL_VALUE) } else if (attr.isMetadataCol) { - new ColumnTypeNode(2) + new ColumnTypeNode(NamedStruct.ColumnType.METADATA_COL_VALUE) } else { - new ColumnTypeNode(0) + new ColumnTypeNode(NamedStruct.ColumnType.NORMAL_COL_VALUE) } }.asJava // Will put all filter expressions into an AND expression @@ -156,8 +157,8 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource TransformContext(output, output, readNode) } - def filteRedundantField(outputs: Seq[Attribute]): Seq[Attribute] = { - var final_output: List[Attribute] = List() + private def filterRedundantField(outputs: Seq[Attribute]): Seq[Attribute] = { + var finalOutput: List[Attribute] = List() val outputList = outputs.toArray for (i <- outputList.indices) { var dup = false @@ -167,9 +168,9 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource } } if (!dup) { - final_output = final_output :+ outputList(i) + finalOutput = finalOutput :+ outputList(i) } } - final_output + finalOutput } } From 945ac2342202533ccf862e248ef262a377ba1569 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 26 Jun 2024 09:57:38 +0800 Subject: [PATCH 334/402] [GLUTEN-6180][VL] Fix NPE if spilling is requested during task creation (#6205) --- .../memory/memtarget/MemoryTargets.java | 2 +- .../arrow/alloc/ArrowBufferAllocators.java | 11 +- .../memory/nmm/NativeMemoryManagers.java | 157 +++++++++--------- .../vectorized/NativePlanEvaluator.java | 2 +- 4 files changed, 91 insertions(+), 81 deletions(-) diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java index 2d6fc0748464..c3ece743310a 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java @@ -63,6 +63,6 @@ public static MemoryTarget newConsumer( factory = TreeMemoryConsumers.shared(); } - return dynamicOffHeapSizingIfEnabled(factory.newConsumer(tmm, name, spillers, virtualChildren)); + return factory.newConsumer(tmm, name, spillers, virtualChildren); } } diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java index efee20e48b83..51f49da704eb 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java @@ -60,11 +60,12 @@ public static class ArrowBufferAllocatorManager implements TaskResource { listener = new ManagedAllocationListener( MemoryTargets.throwOnOom( - MemoryTargets.newConsumer( - tmm, - "ArrowContextInstance", - Collections.emptyList(), - Collections.emptyMap())), + MemoryTargets.dynamicOffHeapSizingIfEnabled( + MemoryTargets.newConsumer( + tmm, + "ArrowContextInstance", + Collections.emptyList(), + Collections.emptyMap()))), TaskResources.getSharedUsage()); } diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java index 928f869ba4e1..37456badd42f 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java @@ -26,6 +26,8 @@ import org.apache.spark.memory.TaskMemoryManager; import org.apache.spark.util.TaskResources; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.Collections; @@ -37,6 +39,7 @@ import java.util.stream.Stream; public final class NativeMemoryManagers { + private static final Logger LOG = LoggerFactory.getLogger(NativeMemoryManagers.class); // TODO: Let all caller support spill. public static NativeMemoryManager contextInstance(String name) { @@ -67,86 +70,92 @@ private static NativeMemoryManager createNativeMemoryManager( final MemoryTarget target = MemoryTargets.throwOnOom( MemoryTargets.overAcquire( - MemoryTargets.newConsumer( - tmm, - name, - // call memory manager's shrink API, if no good then call the spiller - Stream.concat( - Stream.of( - new Spiller() { - @Override - public long spill(MemoryTarget self, long size) { - return Optional.of(out.get()) - .map(nmm -> nmm.shrink(size)) - .orElseThrow( - () -> - new IllegalStateException( - "" - + "Shrink is requested before native " - + "memory manager is created. Try moving " - + "any actions about memory allocation out " - + "from the memory manager constructor.")); - } + MemoryTargets.dynamicOffHeapSizingIfEnabled( + MemoryTargets.newConsumer( + tmm, + name, + // call memory manager's shrink API, if no good then call the spiller + Stream.concat( + Stream.of( + new Spiller() { + @Override + public long spill(MemoryTarget self, long size) { + return Optional.ofNullable(out.get()) + .map(nmm -> nmm.shrink(size)) + .orElseGet( + () -> { + LOG.warn( + "Shrink is requested before native " + + "memory manager is created. Try moving " + + "any actions about memory allocation" + + " out from the memory manager" + + " constructor."); + return 0L; + }); + } - @Override - public Set applicablePhases() { - return Spillers.PHASE_SET_SHRINK_ONLY; - } - }), - spillers.stream()) - .map(spiller -> Spillers.withMinSpillSize(spiller, reservationBlockSize)) - .collect(Collectors.toList()), - Collections.singletonMap( - "single", - new MemoryUsageRecorder() { - @Override - public void inc(long bytes) { - // no-op - } + @Override + public Set applicablePhases() { + return Spillers.PHASE_SET_SHRINK_ONLY; + } + }), + spillers.stream()) + .map( + spiller -> Spillers.withMinSpillSize(spiller, reservationBlockSize)) + .collect(Collectors.toList()), + Collections.singletonMap( + "single", + new MemoryUsageRecorder() { + @Override + public void inc(long bytes) { + // no-op + } - @Override - public long peak() { - throw new UnsupportedOperationException("Not implemented"); - } + @Override + public long peak() { + throw new UnsupportedOperationException("Not implemented"); + } - @Override - public long current() { - throw new UnsupportedOperationException("Not implemented"); - } + @Override + public long current() { + throw new UnsupportedOperationException("Not implemented"); + } - @Override - public MemoryUsageStats toStats() { - return getNativeMemoryManager().collectMemoryUsage(); - } + @Override + public MemoryUsageStats toStats() { + return getNativeMemoryManager().collectMemoryUsage(); + } - private NativeMemoryManager getNativeMemoryManager() { - return Optional.of(out.get()) - .orElseThrow( - () -> - new IllegalStateException( - "" - + "Memory usage stats are requested before native " - + "memory manager is created. Try moving any " - + "actions about memory allocation out from the " - + "memory manager constructor.")); - } - })), - MemoryTargets.newConsumer( - tmm, - "OverAcquire.DummyTarget", - Collections.singletonList( - new Spiller() { - @Override - public long spill(MemoryTarget self, long size) { - return self.repay(size); - } + private NativeMemoryManager getNativeMemoryManager() { + return Optional.ofNullable(out.get()) + .orElseThrow( + () -> + new IllegalStateException( + "" + + "Memory usage stats are requested before" + + " native memory manager is created. Try" + + " moving any actions about memory" + + " allocation out from the memory manager" + + " constructor.")); + } + }))), + MemoryTargets.dynamicOffHeapSizingIfEnabled( + MemoryTargets.newConsumer( + tmm, + "OverAcquire.DummyTarget", + Collections.singletonList( + new Spiller() { + @Override + public long spill(MemoryTarget self, long size) { + return self.repay(size); + } - @Override - public Set applicablePhases() { - return Spillers.PHASE_SET_ALL; - } - }), - Collections.emptyMap()), + @Override + public Set applicablePhases() { + return Spillers.PHASE_SET_ALL; + } + }), + Collections.emptyMap())), overAcquiredRatio)); // listener ManagedReservationListener rl = diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java index e54724a599c1..2ac048b2b960 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java @@ -71,7 +71,7 @@ public GeneralOutIterator createKernelWithBatchIterator( @Override public long spill(MemoryTarget self, long size) { ColumnarBatchOutIterator instance = - Optional.of(outIterator.get()) + Optional.ofNullable(outIterator.get()) .orElseThrow( () -> new IllegalStateException( From 9d2fcdeaaa2631bb50eedf214d6684f9aa2252ce Mon Sep 17 00:00:00 2001 From: Kerwin Zhang Date: Wed, 26 Jun 2024 11:20:19 +0800 Subject: [PATCH 335/402] [CELEBORN] Fix potential ClassNotFoundException (#6217) --- .../org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java index 4593d019c27e..9dd4e1d1191e 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java @@ -65,7 +65,7 @@ public static boolean unregisterShuffle( unregisterAppShuffleId.invoke(shuffleIdTracker, shuffleClient, appShuffleId); } return true; - } catch (NoSuchMethodException ex) { + } catch (NoSuchMethodException | ClassNotFoundException ex) { try { if (lifecycleManager != null) { Method unregisterShuffleMethod = From 529d3f403d2f1b1c9bf6105e0468f4a9f3a19d43 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 26 Jun 2024 13:45:20 +0800 Subject: [PATCH 336/402] [VL] Add a benchmark to track on iterator facility's performance (#6225) --- .../backendsapi/velox/VeloxIteratorApi.scala | 3 +- .../datasource/ArrowCSVFileFormat.scala | 3 +- .../execution/RowToVeloxColumnarExec.scala | 3 +- .../execution/VeloxAppendBatchesExec.scala | 3 +- .../VeloxBroadcastBuildSideRDD.scala | 2 +- .../execution/VeloxColumnarToRowExec.scala | 2 +- .../python/ColumnarArrowEvalPythonExec.scala | 3 +- .../ColumnarCachedBatchSerializer.scala | 3 +- .../datasources/VeloxWriteQueue.scala | 2 +- .../org/apache/gluten/utils/Iterators.scala | 228 ------------------ .../gluten/utils/iterator/Iterators.scala | 53 ++++ .../gluten/utils/iterator/IteratorsV1.scala | 222 +++++++++++++++++ .../utils/{ => iterator}/IteratorSuite.scala | 30 +-- .../utils/iterator/IteratorBenchmark.scala | 129 ++++++++++ .../execution/ColumnarBuildSideRelation.scala | 3 +- .../spark/sql/execution/utils/ExecUtil.scala | 2 +- 16 files changed, 438 insertions(+), 253 deletions(-) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/utils/iterator/Iterators.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala rename gluten-core/src/test/scala/org/apache/gluten/utils/{ => iterator}/IteratorSuite.scala (86%) create mode 100644 gluten-core/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 880e1e56b852..22862156c6b2 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -26,6 +26,7 @@ import org.apache.gluten.substrait.plan.PlanNode import org.apache.gluten.substrait.rel.{LocalFilesBuilder, LocalFilesNode, SplitInfo} import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.gluten.utils._ +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized._ import org.apache.spark.{SparkConf, TaskContext} @@ -36,7 +37,7 @@ import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile} import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.types.{BinaryType, DateType, Decimal, DecimalType, StructType, TimestampType} +import org.apache.spark.sql.types._ import org.apache.spark.sql.utils.OASPackageBridge.InputMetricsWrapper import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.ExecutorManager diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala index 7c3ca8fc8cde..a8e65b0539c7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala @@ -21,7 +21,8 @@ import org.apache.gluten.exception.SchemaMismatchException import org.apache.gluten.execution.RowToVeloxColumnarExec import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool -import org.apache.gluten.utils.{ArrowUtil, Iterators} +import org.apache.gluten.utils.ArrowUtil +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.TaskContext diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 5c9c5889bd13..d694f15fa9bd 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -22,7 +22,8 @@ import org.apache.gluten.exception.GlutenException import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers -import org.apache.gluten.utils.{ArrowAbiUtil, Iterators} +import org.apache.gluten.utils.ArrowAbiUtil +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized._ import org.apache.spark.broadcast.Broadcast diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala index 8c2834574204..4b4db703de7a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxAppendBatchesExec.scala @@ -17,7 +17,8 @@ package org.apache.gluten.execution import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.utils.{Iterators, VeloxBatchAppender} +import org.apache.gluten.utils.VeloxBatchAppender +import org.apache.gluten.utils.iterator.Iterators import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastBuildSideRDD.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastBuildSideRDD.scala index 17d0522d0732..fe3c0b7e3938 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastBuildSideRDD.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastBuildSideRDD.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.utils.Iterators +import org.apache.gluten.utils.iterator.Iterators import org.apache.spark.{broadcast, SparkContext} import org.apache.spark.sql.execution.joins.BuildSideRelation diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala index 77bf49727283..0d6714d3af92 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala @@ -20,7 +20,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.extension.ValidationResult import org.apache.gluten.memory.nmm.NativeMemoryManagers -import org.apache.gluten.utils.Iterators +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.NativeColumnarToRowJniWrapper import org.apache.spark.broadcast.Broadcast diff --git a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala index d5639057dac8..88280ff2edde 100644 --- a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala +++ b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala @@ -20,7 +20,8 @@ import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenException import org.apache.gluten.extension.GlutenPlan import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.utils.{Iterators, PullOutProjectHelper} +import org.apache.gluten.utils.PullOutProjectHelper +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.ArrowWritableColumnVector import org.apache.spark.{ContextAwareIterator, SparkEnv, TaskContext} diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index 7385c53d61b3..cb65b7504bfc 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -23,7 +23,8 @@ import org.apache.gluten.exec.Runtimes import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers -import org.apache.gluten.utils.{ArrowAbiUtil, Iterators} +import org.apache.gluten.utils.ArrowAbiUtil +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper import org.apache.spark.internal.Logging diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/VeloxWriteQueue.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/VeloxWriteQueue.scala index 089db1da1dee..b2905e157554 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/VeloxWriteQueue.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/VeloxWriteQueue.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.datasources import org.apache.gluten.datasource.DatasourceJniWrapper -import org.apache.gluten.utils.Iterators +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.ColumnarBatchInIterator import org.apache.spark.TaskContext diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala deleted file mode 100644 index 1e3681355d6c..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/Iterators.scala +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils - -import org.apache.spark.{InterruptibleIterator, TaskContext} -import org.apache.spark.util.TaskResources - -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean - -private class PayloadCloser[A](in: Iterator[A])(closeCallback: A => Unit) extends Iterator[A] { - private var closer: Option[() => Unit] = None - - TaskResources.addRecycler("Iterators#PayloadCloser", 100) { - tryClose() - } - - override def hasNext: Boolean = { - tryClose() - in.hasNext - } - - override def next(): A = { - val a: A = in.next() - closer.synchronized { - closer = Some( - () => { - closeCallback.apply(a) - }) - } - a - } - - private def tryClose(): Unit = { - closer.synchronized { - closer match { - case Some(c) => c.apply() - case None => - } - closer = None // make sure the payload is closed once - } - } -} - -private class IteratorCompleter[A](in: Iterator[A])(completionCallback: => Unit) - extends Iterator[A] { - private val completed = new AtomicBoolean(false) - - TaskResources.addRecycler("Iterators#IteratorRecycler", 100) { - tryComplete() - } - - override def hasNext: Boolean = { - val out = in.hasNext - if (!out) { - tryComplete() - } - out - } - - override def next(): A = { - in.next() - } - - private def tryComplete(): Unit = { - if (!completed.compareAndSet(false, true)) { - return // make sure the iterator is completed once - } - completionCallback - } -} - -private class LifeTimeAccumulator[A](in: Iterator[A], onCollected: Long => Unit) - extends Iterator[A] { - private val closed = new AtomicBoolean(false) - private val startTime = System.nanoTime() - - TaskResources.addRecycler("Iterators#LifeTimeAccumulator", 100) { - tryFinish() - } - - override def hasNext: Boolean = { - val out = in.hasNext - if (!out) { - tryFinish() - } - out - } - - override def next(): A = { - in.next() - } - - private def tryFinish(): Unit = { - // pipeline metric should only be calculate once. - if (!closed.compareAndSet(false, true)) { - return - } - val lifeTime = TimeUnit.NANOSECONDS.toMillis( - System.nanoTime() - startTime - ) - onCollected(lifeTime) - } -} - -private class ReadTimeAccumulator[A](in: Iterator[A], onAdded: Long => Unit) extends Iterator[A] { - - override def hasNext: Boolean = { - val prev = System.nanoTime() - val out = in.hasNext - val after = System.nanoTime() - val duration = TimeUnit.NANOSECONDS.toMillis(after - prev) - onAdded(duration) - out - } - - override def next(): A = { - val prev = System.nanoTime() - val out = in.next() - val after = System.nanoTime() - val duration = TimeUnit.NANOSECONDS.toMillis(after - prev) - onAdded(duration) - out - } -} - -/** - * To protect the wrapped iterator to avoid undesired order of calls to its `hasNext` and `next` - * methods. - */ -private class InvocationFlowProtection[A](in: Iterator[A]) extends Iterator[A] { - sealed private trait State - private case object Init extends State - private case class HasNextCalled(hasNext: Boolean) extends State - private case object NextCalled extends State - - private var state: State = Init - - override def hasNext: Boolean = { - val out = state match { - case Init | NextCalled => - in.hasNext - case HasNextCalled(lastHasNext) => - lastHasNext - } - state = HasNextCalled(out) - out - } - - override def next(): A = { - val out = state match { - case Init | NextCalled => - if (!in.hasNext) { - throw new IllegalStateException("End of stream") - } - in.next() - case HasNextCalled(lastHasNext) => - if (!lastHasNext) { - throw new IllegalStateException("End of stream") - } - in.next() - } - state = NextCalled - out - } -} - -class WrapperBuilder[A](in: Iterator[A]) { // FIXME how to make the ctor companion-private? - private var wrapped: Iterator[A] = in - - def recyclePayload(closeCallback: (A) => Unit): WrapperBuilder[A] = { - wrapped = new PayloadCloser(wrapped)(closeCallback) - this - } - - def recycleIterator(completionCallback: => Unit): WrapperBuilder[A] = { - wrapped = new IteratorCompleter(wrapped)(completionCallback) - this - } - - def collectLifeMillis(onCollected: Long => Unit): WrapperBuilder[A] = { - wrapped = new LifeTimeAccumulator[A](wrapped, onCollected) - this - } - - def collectReadMillis(onAdded: Long => Unit): WrapperBuilder[A] = { - wrapped = new ReadTimeAccumulator[A](wrapped, onAdded) - this - } - - def asInterruptible(context: TaskContext): WrapperBuilder[A] = { - wrapped = new InterruptibleIterator[A](context, wrapped) - this - } - - def protectInvocationFlow(): WrapperBuilder[A] = { - wrapped = new InvocationFlowProtection[A](wrapped) - this - } - - def create(): Iterator[A] = { - wrapped - } -} - -/** - * Utility class to provide iterator wrappers for non-trivial use cases. E.g. iterators that manage - * payload's lifecycle. - */ -object Iterators { - def wrap[A](in: Iterator[A]): WrapperBuilder[A] = { - new WrapperBuilder[A](in) - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/iterator/Iterators.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/iterator/Iterators.scala new file mode 100644 index 000000000000..eedfa66cfeaf --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/iterator/Iterators.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.utils.iterator + +import org.apache.gluten.utils.iterator.IteratorsV1.WrapperBuilderV1 + +import org.apache.spark.TaskContext + +/** + * Utility class to provide iterator wrappers for non-trivial use cases. E.g. iterators that manage + * payload's lifecycle. + */ +object Iterators { + sealed trait Version + case object V1 extends Version + + private val DEFAULT_VERSION: Version = V1 + + trait WrapperBuilder[A] { + def recyclePayload(closeCallback: (A) => Unit): WrapperBuilder[A] + def recycleIterator(completionCallback: => Unit): WrapperBuilder[A] + def collectLifeMillis(onCollected: Long => Unit): WrapperBuilder[A] + def collectReadMillis(onAdded: Long => Unit): WrapperBuilder[A] + def asInterruptible(context: TaskContext): WrapperBuilder[A] + def protectInvocationFlow(): WrapperBuilder[A] + def create(): Iterator[A] + } + + def wrap[A](in: Iterator[A]): WrapperBuilder[A] = { + wrap(V1, in) + } + + def wrap[A](version: Version, in: Iterator[A]): WrapperBuilder[A] = { + version match { + case V1 => + new WrapperBuilderV1[A](in) + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala new file mode 100644 index 000000000000..3e9248c44458 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.utils.iterator + +import org.apache.gluten.utils.iterator.Iterators.WrapperBuilder + +import org.apache.spark.{InterruptibleIterator, TaskContext} +import org.apache.spark.util.TaskResources + +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +object IteratorsV1 { + private class PayloadCloser[A](in: Iterator[A])(closeCallback: A => Unit) extends Iterator[A] { + private var closer: Option[() => Unit] = None + + TaskResources.addRecycler("Iterators#PayloadCloser", 100) { + tryClose() + } + + override def hasNext: Boolean = { + tryClose() + in.hasNext + } + + override def next(): A = { + val a: A = in.next() + closer.synchronized { + closer = Some( + () => { + closeCallback.apply(a) + }) + } + a + } + + private def tryClose(): Unit = { + closer.synchronized { + closer match { + case Some(c) => c.apply() + case None => + } + closer = None // make sure the payload is closed once + } + } + } + + private class IteratorCompleter[A](in: Iterator[A])(completionCallback: => Unit) + extends Iterator[A] { + private val completed = new AtomicBoolean(false) + + TaskResources.addRecycler("Iterators#IteratorRecycler", 100) { + tryComplete() + } + + override def hasNext: Boolean = { + val out = in.hasNext + if (!out) { + tryComplete() + } + out + } + + override def next(): A = { + in.next() + } + + private def tryComplete(): Unit = { + if (!completed.compareAndSet(false, true)) { + return // make sure the iterator is completed once + } + completionCallback + } + } + + private class LifeTimeAccumulator[A](in: Iterator[A], onCollected: Long => Unit) + extends Iterator[A] { + private val closed = new AtomicBoolean(false) + private val startTime = System.nanoTime() + + TaskResources.addRecycler("Iterators#LifeTimeAccumulator", 100) { + tryFinish() + } + + override def hasNext: Boolean = { + val out = in.hasNext + if (!out) { + tryFinish() + } + out + } + + override def next(): A = { + in.next() + } + + private def tryFinish(): Unit = { + // pipeline metric should only be calculate once. + if (!closed.compareAndSet(false, true)) { + return + } + val lifeTime = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - startTime + ) + onCollected(lifeTime) + } + } + + private class ReadTimeAccumulator[A](in: Iterator[A], onAdded: Long => Unit) extends Iterator[A] { + + override def hasNext: Boolean = { + val prev = System.nanoTime() + val out = in.hasNext + val after = System.nanoTime() + val duration = TimeUnit.NANOSECONDS.toMillis(after - prev) + onAdded(duration) + out + } + + override def next(): A = { + val prev = System.nanoTime() + val out = in.next() + val after = System.nanoTime() + val duration = TimeUnit.NANOSECONDS.toMillis(after - prev) + onAdded(duration) + out + } + } + + /** + * To protect the wrapped iterator to avoid undesired order of calls to its `hasNext` and `next` + * methods. + */ + private class InvocationFlowProtection[A](in: Iterator[A]) extends Iterator[A] { + sealed private trait State + private case object Init extends State + private case class HasNextCalled(hasNext: Boolean) extends State + private case object NextCalled extends State + + private var state: State = Init + + override def hasNext: Boolean = { + val out = state match { + case Init | NextCalled => + in.hasNext + case HasNextCalled(lastHasNext) => + lastHasNext + } + state = HasNextCalled(out) + out + } + + override def next(): A = { + val out = state match { + case Init | NextCalled => + if (!in.hasNext) { + throw new IllegalStateException("End of stream") + } + in.next() + case HasNextCalled(lastHasNext) => + if (!lastHasNext) { + throw new IllegalStateException("End of stream") + } + in.next() + } + state = NextCalled + out + } + } + + class WrapperBuilderV1[A] private[iterator] (in: Iterator[A]) extends WrapperBuilder[A] { + private var wrapped: Iterator[A] = in + + override def recyclePayload(closeCallback: (A) => Unit): WrapperBuilder[A] = { + wrapped = new PayloadCloser(wrapped)(closeCallback) + this + } + + override def recycleIterator(completionCallback: => Unit): WrapperBuilder[A] = { + wrapped = new IteratorCompleter(wrapped)(completionCallback) + this + } + + override def collectLifeMillis(onCollected: Long => Unit): WrapperBuilder[A] = { + wrapped = new LifeTimeAccumulator[A](wrapped, onCollected) + this + } + + override def collectReadMillis(onAdded: Long => Unit): WrapperBuilder[A] = { + wrapped = new ReadTimeAccumulator[A](wrapped, onAdded) + this + } + + override def asInterruptible(context: TaskContext): WrapperBuilder[A] = { + wrapped = new InterruptibleIterator[A](context, wrapped) + this + } + + override def protectInvocationFlow(): WrapperBuilder[A] = { + wrapped = new InvocationFlowProtection[A](wrapped) + this + } + + override def create(): Iterator[A] = { + wrapped + } + } +} diff --git a/gluten-core/src/test/scala/org/apache/gluten/utils/IteratorSuite.scala b/gluten-core/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala similarity index 86% rename from gluten-core/src/test/scala/org/apache/gluten/utils/IteratorSuite.scala rename to gluten-core/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala index 389e2adfefd4..1a84d671922d 100644 --- a/gluten-core/src/test/scala/org/apache/gluten/utils/IteratorSuite.scala +++ b/gluten-core/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala @@ -14,18 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.utils +package org.apache.gluten.utils.iterator + +import org.apache.gluten.utils.iterator.Iterators.{V1, WrapperBuilder} import org.apache.spark.util.TaskResources import org.scalatest.funsuite.AnyFunSuite -class IteratorSuite extends AnyFunSuite { +class IteratorV1Suite extends IteratorSuite { + override protected def wrap[A](in: Iterator[A]): WrapperBuilder[A] = Iterators.wrap(V1, in) +} + +abstract class IteratorSuite extends AnyFunSuite { + protected def wrap[A](in: Iterator[A]): WrapperBuilder[A] + test("Trivial wrapping") { val strings = Array[String]("one", "two", "three") val itr = strings.toIterator - val wrapped = Iterators - .wrap(itr) + val wrapped = wrap(itr) .create() assertResult(strings) { wrapped.toArray @@ -37,8 +44,7 @@ class IteratorSuite extends AnyFunSuite { TaskResources.runUnsafe { val strings = Array[String]("one", "two", "three") val itr = strings.toIterator - val wrapped = Iterators - .wrap(itr) + val wrapped = wrap(itr) .recycleIterator { completeCount += 1 } @@ -56,8 +62,7 @@ class IteratorSuite extends AnyFunSuite { TaskResources.runUnsafe { val strings = Array[String]("one", "two", "three") val itr = strings.toIterator - val _ = Iterators - .wrap(itr) + val _ = wrap(itr) .recycleIterator { completeCount += 1 } @@ -72,8 +77,7 @@ class IteratorSuite extends AnyFunSuite { TaskResources.runUnsafe { val strings = Array[String]("one", "two", "three") val itr = strings.toIterator - val wrapped = Iterators - .wrap(itr) + val wrapped = wrap(itr) .recyclePayload { _: String => closeCount += 1 } .create() assertResult(strings) { @@ -89,8 +93,7 @@ class IteratorSuite extends AnyFunSuite { TaskResources.runUnsafe { val strings = Array[String]("one", "two", "three") val itr = strings.toIterator - val wrapped = Iterators - .wrap(itr) + val wrapped = wrap(itr) .recyclePayload { _: String => closeCount += 1 } .create() assertResult(strings.take(2)) { @@ -115,8 +118,7 @@ class IteratorSuite extends AnyFunSuite { new Object } } - val wrapped = Iterators - .wrap(itr) + val wrapped = wrap(itr) .protectInvocationFlow() .create() wrapped.hasNext diff --git a/gluten-core/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala b/gluten-core/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala new file mode 100644 index 000000000000..aa69f309aac8 --- /dev/null +++ b/gluten-core/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.utils.iterator + +import org.apache.gluten.utils.iterator.Iterators +import org.apache.gluten.utils.iterator.Iterators.V1 + +import org.apache.spark.benchmark.{Benchmark, BenchmarkBase} +import org.apache.spark.util.TaskResources + +object IteratorBenchmark extends BenchmarkBase { + + override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { + runBenchmark("Iterator Nesting") { + TaskResources.runUnsafe { + val nPayloads: Int = 50000000 // 50 millions + + def makeScalaIterator: Iterator[Any] = { + (0 until nPayloads).view.map { _: Int => new Object }.iterator + } + + def compareIterator(name: String)( + makeGlutenIterator: Iterators.Version => Iterator[Any]): Unit = { + val benchmark = new Benchmark(name, nPayloads, output = output) + benchmark.addCase("Scala Iterator") { + _ => + val count = makeScalaIterator.count(_ => true) + assert(count == nPayloads) + } + benchmark.addCase("Gluten Iterator V1") { + _ => + val count = makeGlutenIterator(V1).count(_ => true) + assert(count == nPayloads) + } + benchmark.run() + } + + compareIterator("0 Levels Nesting") { + version => + Iterators + .wrap(version, makeScalaIterator) + .create() + } + compareIterator("1 Levels Nesting - read") { + version => + Iterators + .wrap(version, makeScalaIterator) + .collectReadMillis { _ => } + .create() + } + compareIterator("5 Levels Nesting - read") { + version => + Iterators + .wrap(version, makeScalaIterator) + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .create() + } + compareIterator("10 Levels Nesting - read") { + version => + Iterators + .wrap(version, makeScalaIterator) + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .collectReadMillis { _ => } + .create() + } + compareIterator("1 Levels Nesting - recycle") { + version => + Iterators + .wrap(version, makeScalaIterator) + .recycleIterator {} + .create() + } + compareIterator("5 Levels Nesting - recycle") { + version => + Iterators + .wrap(version, makeScalaIterator) + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .create() + } + compareIterator("10 Levels Nesting - recycle") { + version => + Iterators + .wrap(version, makeScalaIterator) + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .recycleIterator {} + .create() + } + } + } + } +} diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index 9d9f5ab1765c..840f8618b0b4 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -21,7 +21,8 @@ import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.sql.shims.SparkShimLoader -import org.apache.gluten.utils.{ArrowAbiUtil, Iterators} +import org.apache.gluten.utils.ArrowAbiUtil +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.{ColumnarBatchSerializerJniWrapper, NativeColumnarToRowJniWrapper} import org.apache.spark.sql.catalyst.InternalRow diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index 083915f12db9..090b8fa2562a 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.utils import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.memory.nmm.NativeMemoryManagers -import org.apache.gluten.utils.Iterators +import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning} import org.apache.spark.{Partitioner, RangePartitioner, ShuffleDependency} From 9ae34a91379fc833c3db873292e53e589be9e62b Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 26 Jun 2024 15:17:12 +0800 Subject: [PATCH 337/402] [GLUTEN-5643] Fix the failure when the pre-project of GenerateExec falls back (#6167) --- .../velox/VeloxSparkPlanExecApi.scala | 4 +- .../gluten/expression/DummyExpression.scala | 77 +++++++++++++++++++ .../spark/sql/expression/UDFResolver.scala | 5 +- .../gluten/execution/TestOperator.scala | 24 +++++- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 38 +++++---- .../expression/ExpressionConverter.scala | 2 +- 6 files changed, 131 insertions(+), 19 deletions(-) create mode 100644 backends-velox/src/main/scala/org/apache/gluten/expression/DummyExpression.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 7b8d523a6d27..b48da15683e8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -852,7 +852,9 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { Sig[VeloxBloomFilterMightContain](ExpressionNames.MIGHT_CONTAIN), Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG), Sig[TransformKeys](TRANSFORM_KEYS), - Sig[TransformValues](TRANSFORM_VALUES) + Sig[TransformValues](TRANSFORM_VALUES), + // For test purpose. + Sig[VeloxDummyExpression](VeloxDummyExpression.VELOX_DUMMY_EXPRESSION) ) } diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/DummyExpression.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/DummyExpression.scala new file mode 100644 index 000000000000..e2af66b599d3 --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/DummyExpression.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.expression + +import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow} +import org.apache.spark.sql.catalyst.analysis.FunctionRegistry +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, UnaryExpression} +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.types.DataType + +abstract class DummyExpression(child: Expression) extends UnaryExpression with Serializable { + private val accessor: (InternalRow, Int) => Any = InternalRow.getAccessor(dataType, nullable) + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = + defineCodeGen(ctx, ev, c => c) + + override def dataType: DataType = child.dataType + + override def eval(input: InternalRow): Any = { + assert(input.numFields == 1, "The input row of DummyExpression should have only 1 field.") + accessor(input, 0) + } +} + +// Can be used as a wrapper to force fall back the original expression to mock the fallback behavior +// of an supported expression in Gluten which fails native validation. +case class VeloxDummyExpression(child: Expression) + extends DummyExpression(child) + with Transformable { + override def getTransformer( + childrenTransformers: Seq[ExpressionTransformer]): ExpressionTransformer = { + if (childrenTransformers.size != children.size) { + throw new IllegalStateException( + this.getClass.getSimpleName + + ": getTransformer called before children transformer initialized.") + } + + GenericExpressionTransformer( + VeloxDummyExpression.VELOX_DUMMY_EXPRESSION, + childrenTransformers, + this) + } + + override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild) +} + +object VeloxDummyExpression { + val VELOX_DUMMY_EXPRESSION = "velox_dummy_expression" + + private val identifier = new FunctionIdentifier(VELOX_DUMMY_EXPRESSION) + + def registerFunctions(registry: FunctionRegistry): Unit = { + registry.registerFunction( + identifier, + new ExpressionInfo(classOf[VeloxDummyExpression].getName, VELOX_DUMMY_EXPRESSION), + (e: Seq[Expression]) => VeloxDummyExpression(e.head) + ) + } + + def unregisterFunctions(registry: FunctionRegistry): Unit = { + registry.dropFunction(identifier) + } +} diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index 915fc554584c..e45e8b6fa6d7 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -27,7 +27,7 @@ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow} import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionInfo} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionInfo, Unevaluable} import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} import org.apache.spark.sql.catalyst.types.DataTypeUtils @@ -94,7 +94,8 @@ case class UDFExpression( dataType: DataType, nullable: Boolean, children: Seq[Expression]) - extends Transformable { + extends Unevaluable + with Transformable { override protected def withNewChildrenInternal( newChildren: IndexedSeq[Expression]): Expression = { this.copy(children = newChildren) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index a892b6f313a4..9b47a519cd28 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -19,6 +19,7 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.datasource.ArrowCSVFileFormat import org.apache.gluten.execution.datasource.v2.ArrowBatchScanExec +import org.apache.gluten.expression.VeloxDummyExpression import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf @@ -45,6 +46,12 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla override def beforeAll(): Unit = { super.beforeAll() createTPCHNotNullTables() + VeloxDummyExpression.registerFunctions(spark.sessionState.functionRegistry) + } + + override def afterAll(): Unit = { + VeloxDummyExpression.unregisterFunctions(spark.sessionState.functionRegistry) + super.afterAll() } override protected def sparkConf: SparkConf = { @@ -66,14 +73,20 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla test("select_part_column") { val df = runQueryAndCompare("select l_shipdate, l_orderkey from lineitem limit 1") { - df => { assert(df.schema.fields.length == 2) } + df => + { + assert(df.schema.fields.length == 2) + } } checkLengthAndPlan(df, 1) } test("select_as") { val df = runQueryAndCompare("select l_shipdate as my_col from lineitem limit 1") { - df => { assert(df.schema.fieldNames(0).equals("my_col")) } + df => + { + assert(df.schema.fieldNames(0).equals("my_col")) + } } checkLengthAndPlan(df, 1) } @@ -1074,6 +1087,13 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla // No ProjectExecTransformer is introduced. checkSparkOperatorChainMatch[GenerateExecTransformer, FilterExecTransformer] } + + runQueryAndCompare( + s""" + |SELECT $func(${VeloxDummyExpression.VELOX_DUMMY_EXPRESSION}(a)) from t2; + |""".stripMargin) { + checkGlutenOperatorMatch[GenerateExecTransformer] + } } } } diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 8b8a9262403c..73047b2f4907 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -26,6 +26,7 @@ #include "utils/ConfigExtractor.h" #include "config/GlutenConfig.h" +#include "operators/plannodes/RowVectorStream.h" namespace gluten { namespace { @@ -710,16 +711,23 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: namespace { void extractUnnestFieldExpr( - std::shared_ptr projNode, + std::shared_ptr child, int32_t index, std::vector& unnestFields) { - auto name = projNode->names()[index]; - auto expr = projNode->projections()[index]; - auto type = expr->type(); + if (auto projNode = std::dynamic_pointer_cast(child)) { + auto name = projNode->names()[index]; + auto expr = projNode->projections()[index]; + auto type = expr->type(); - auto unnestFieldExpr = std::make_shared(type, name); - VELOX_CHECK_NOT_NULL(unnestFieldExpr, " the key in unnest Operator only support field"); - unnestFields.emplace_back(unnestFieldExpr); + auto unnestFieldExpr = std::make_shared(type, name); + VELOX_CHECK_NOT_NULL(unnestFieldExpr, " the key in unnest Operator only support field"); + unnestFields.emplace_back(unnestFieldExpr); + } else { + auto name = child->outputType()->names()[index]; + auto field = child->outputType()->childAt(index); + auto unnestFieldExpr = std::make_shared(field, name); + unnestFields.emplace_back(unnestFieldExpr); + } } } // namespace @@ -752,10 +760,13 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: SubstraitParser::configSetInOptimization(generateRel.advanced_extension(), "injectedProject="); if (injectedProject) { - auto projNode = std::dynamic_pointer_cast(childNode); + // Child should be either ProjectNode or ValueStreamNode in case of project fallback. VELOX_CHECK( - projNode != nullptr && projNode->names().size() > requiredChildOutput.size(), - "injectedProject is true, but the Project is missing or does not have the corresponding projection field") + (std::dynamic_pointer_cast(childNode) != nullptr || + std::dynamic_pointer_cast(childNode) != nullptr) && + childNode->outputType()->size() > requiredChildOutput.size(), + "injectedProject is true, but the ProjectNode or ValueStreamNode (in case of projection fallback)" + " is missing or does not have the corresponding projection field") bool isStack = generateRel.has_advanced_extension() && SubstraitParser::configSetInOptimization(generateRel.advanced_extension(), "isStack="); @@ -768,7 +779,8 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: // +- Project [fake_column#128, [1,2,3] AS _pre_0#129] // +- RewrittenNodeWall Scan OneRowRelation[fake_column#128] // The last projection column in GeneratorRel's child(Project) is the column we need to unnest - extractUnnestFieldExpr(projNode, projNode->projections().size() - 1, unnest); + auto index = childNode->outputType()->size() - 1; + extractUnnestFieldExpr(childNode, index, unnest); } else { // For stack function, e.g. stack(2, 1,2,3), a sample // input substrait plan is like the following: @@ -782,10 +794,10 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: auto generatorFunc = generator.scalar_function(); auto numRows = SubstraitParser::getLiteralValue(generatorFunc.arguments(0).value().literal()); auto numFields = static_cast(std::ceil((generatorFunc.arguments_size() - 1.0) / numRows)); - auto totalProjectCount = projNode->names().size(); + auto totalProjectCount = childNode->outputType()->size(); for (auto i = totalProjectCount - numFields; i < totalProjectCount; ++i) { - extractUnnestFieldExpr(projNode, i, unnest); + extractUnnestFieldExpr(childNode, i, unnest); } } } else { diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index b7b0889dc1eb..da5625cd45e5 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.hive.HiveUDFTransformer import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -trait Transformable extends Unevaluable { +trait Transformable { def getTransformer(childrenTransformers: Seq[ExpressionTransformer]): ExpressionTransformer } From e30006464e507744a7e433718f5778bb2d58856f Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 26 Jun 2024 15:36:36 +0800 Subject: [PATCH 338/402] [VL] Daily Update Velox Version (2024_06_26) (#6223) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 06998787d45e..a96719dc10fc 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_25 +VELOX_BRANCH=2024_06_26 VELOX_HOME="" #Set on run gluten on HDFS From a51f6931007256eec10b1a7ae69ef39554ce52f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Wed, 26 Jun 2024 15:51:18 +0800 Subject: [PATCH 339/402] [GLUTEN-6208][CH] Enable more uts in GlutenStringExpressionsSuite (#6218) --- .../Parser/SerializedPlanParser.cpp | 13 --- .../Parser/SerializedPlanParser.h | 2 - .../Parser/scalar_function_parser/concat.cpp | 79 +++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 9 --- .../clickhouse/ClickHouseTestSettings.scala | 61 -------------- .../clickhouse/ClickHouseTestSettings.scala | 12 --- .../clickhouse/ClickHouseTestSettings.scala | 12 --- 7 files changed, 79 insertions(+), 109 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/concat.cpp diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 3115950cdf09..325ec32dc65f 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -664,19 +664,6 @@ SerializedPlanParser::getFunctionName(const std::string & function_signature, co else ch_function_name = "reverseUTF8"; } - else if (function_name == "concat") - { - /// 1. ConcatOverloadResolver cannot build arrayConcat for Nullable(Array) type which causes failures when using functions like concat(split()). - /// So we use arrayConcat directly if the output type is array. - /// 2. CH ConcatImpl can only accept at least 2 arguments, but Spark concat can accept 1 argument, like concat('a') - /// in such case we use identity function - if (function.output_type().has_list()) - ch_function_name = "arrayConcat"; - else if (args.size() == 1) - ch_function_name = "identity"; - else - ch_function_name = "concat"; - } else ch_function_name = SCALAR_FUNCTIONS.at(function_name); diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index aa18197e5647..6ce92b558b73 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -127,13 +127,11 @@ static const std::map SCALAR_FUNCTIONS {"trim", ""}, // trimLeft or trimLeftSpark, depends on argument size {"ltrim", ""}, // trimRight or trimRightSpark, depends on argument size {"rtrim", ""}, // trimBoth or trimBothSpark, depends on argument size - {"concat", ""}, /// dummy mapping {"strpos", "positionUTF8"}, {"char_length", "char_length"}, /// Notice: when input argument is binary type, corresponding ch function is length instead of char_length {"replace", "replaceAll"}, {"regexp_replace", "replaceRegexpAll"}, - // {"regexp_extract", "regexpExtract"}, {"regexp_extract_all", "regexpExtractAllSpark"}, {"chr", "char"}, {"rlike", "match"}, diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/concat.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/concat.cpp new file mode 100644 index 000000000000..416fe7741812 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/concat.cpp @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ + +class FunctionParserConcat : public FunctionParser +{ +public: + explicit FunctionParserConcat(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~FunctionParserConcat() override = default; + + static constexpr auto name = "concat"; + + String getName() const override { return name; } + + const ActionsDAG::Node * parse( + const substrait::Expression_ScalarFunction & substrait_func, + ActionsDAGPtr & actions_dag) const override + { + /* + parse concat(args) as: + 1. if output type is array, return arrayConcat(args) + 2. otherwise: + 1) if args is empty, return empty string + 2) if args have size 1, return identity(args[0]) + 3) otherwise return concat(args) + */ + auto args = parseFunctionArguments(substrait_func, "", actions_dag); + const auto & output_type = substrait_func.output_type(); + const ActionsDAG::Node * result_node = nullptr; + if (output_type.has_list()) + { + result_node = toFunctionNode(actions_dag, "arrayConcat", args); + } + else + { + if (args.empty()) + result_node = addColumnToActionsDAG(actions_dag, std::make_shared(), ""); + else if (args.size() == 1) + result_node = toFunctionNode(actions_dag, "identity", args); + else + result_node = toFunctionNode(actions_dag, "concat", args); + } + return convertNodeTypeIfNeeded(substrait_func, result_node, actions_dag); + } +}; + +static FunctionParserRegister register_concat; +} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 1626716805cb..d12a40b764f8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -437,7 +437,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string regex_replace / regex_extract") .exclude("string overlay function") .exclude("binary overlay function") - .exclude("string / binary substring function") .exclude("string parse_url function") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") @@ -894,7 +893,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-34814: LikeSimplification should handle NULL") enableSuite[GlutenSortOrderExpressionsSuite].exclude("SortPrefix") enableSuite[GlutenStringExpressionsSuite] - .exclude("concat") .exclude("StringComparison") .exclude("Substring") .exclude("string substring_index function") @@ -902,22 +900,15 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") - .exclude("soundex unit test") - .exclude("replace") .exclude("overlay for string") .exclude("overlay for byte array") .exclude("translate") - .exclude("FORMAT") - .exclude("SPARK-22603: FormatString should not generate codes beyond 64KB") - .exclude("INSTR") .exclude("LOCATE") .exclude("LPAD/RPAD") .exclude("REPEAT") .exclude("length for string / binary") - .exclude("format_number / FormatNumber") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") - .exclude("Sentences") .excludeGlutenTest("SPARK-40213: ascii for Latin-1 Supplement characters") enableSuite[GlutenTryCastSuite] .exclude("null cast") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 3147c7c3dbf3..52e7ebcbda49 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -458,7 +458,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string regex_replace / regex_extract") .exclude("string overlay function") .exclude("binary overlay function") - .exclude("string / binary substring function") .exclude("string parse_url function") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") @@ -474,58 +473,9 @@ class ClickHouseTestSettings extends BackendTestSettings { enableSuite[GlutenXPathFunctionsSuite] enableSuite[QueryTestSuite] enableSuite[GlutenAnsiCastSuiteWithAnsiModeOff] - .exclude("null cast") .exclude("cast string to date") - .exclude("cast string to timestamp") - .exclude("cast from boolean") - .exclude("cast from int") - .exclude("cast from long") - .exclude("cast from float") - .exclude("cast from double") - .exclude("cast from timestamp") - .exclude("data type casting") - .exclude("cast and add") - .exclude("from decimal") - .exclude("cast from array") - .exclude("cast from map") - .exclude("cast from struct") - .exclude("cast struct with a timestamp field") - .exclude("cast between string and interval") - .exclude("cast string to boolean") - .exclude("SPARK-20302 cast with same structure") - .exclude("SPARK-22500: cast for struct should not generate codes beyond 64KB") - .exclude("SPARK-27671: cast from nested null type in struct") - .exclude("Process Infinity, -Infinity, NaN in case insensitive manner") - .exclude("SPARK-22825 Cast array to string") - .exclude("SPARK-33291: Cast array with null elements to string") - .exclude("SPARK-22973 Cast map to string") - .exclude("SPARK-22981 Cast struct to string") - .exclude("SPARK-33291: Cast struct with null elements to string") - .exclude("SPARK-34667: cast year-month interval to string") - .exclude("SPARK-34668: cast day-time interval to string") - .exclude("SPARK-35698: cast timestamp without time zone to string") .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") - .exclude("SPARK-35716: cast timestamp without time zone to date type") - .exclude("SPARK-35718: cast date type to timestamp without timezone") - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") - .exclude("SPARK-35720: cast string to timestamp without timezone") - .exclude("SPARK-35112: Cast string to day-time interval") - .exclude("SPARK-35111: Cast string to year-month interval") - .exclude("SPARK-35820: Support cast DayTimeIntervalType in different fields") .exclude("SPARK-35819: Support cast YearMonthIntervalType in different fields") - .exclude("SPARK-35768: Take into account year-month interval fields in cast") - .exclude("SPARK-35735: Take into account day-time interval fields in cast") - .exclude("ANSI mode: Throw exception on casting out-of-range value to byte type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to short type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to int type") - .exclude("ANSI mode: Throw exception on casting out-of-range value to long type") - .exclude("Fast fail for cast string type to decimal type in ansi mode") - .exclude("cast a timestamp before the epoch 1970-01-01 00:00:00Z") - .exclude("cast from array III") - .exclude("cast from map II") - .exclude("cast from map III") - .exclude("cast from struct II") - .exclude("cast from struct III") enableSuite[GlutenAnsiCastSuiteWithAnsiModeOn] .exclude("null cast") .exclude("cast string to date") @@ -902,7 +852,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK - 34814: LikeSimplification should handleNULL") enableSuite[GlutenSortOrderExpressionsSuite].exclude("SortPrefix") enableSuite[GlutenStringExpressionsSuite] - .exclude("concat") .exclude("StringComparison") .exclude("Substring") .exclude("string substring_index function") @@ -911,24 +860,14 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") - .exclude("soundex unit test") - .exclude("replace") .exclude("overlay for string") .exclude("overlay for byte array") .exclude("translate") - .exclude("FORMAT") - .exclude("SPARK-22603: FormatString should not generate codes beyond 64KB") - .exclude("INSTR") .exclude("LOCATE") - .exclude("LPAD/RPAD") .exclude("REPEAT") .exclude("length for string / binary") - .exclude("format_number / FormatNumber") - .exclude("ToNumber: positive tests") - .exclude("ToNumber: negative tests (the input string does not match the format string)") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") - .exclude("Sentences") enableSuite[GlutenTryCastSuite] .exclude("null cast") .exclude("cast string to date") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 07af1fa845ca..38ed2c53463b 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -457,7 +457,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string regex_replace / regex_extract") .exclude("string overlay function") .exclude("binary overlay function") - .exclude("string / binary substring function") .exclude("string parse_url function") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") @@ -756,7 +755,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK - 34814: LikeSimplification should handleNULL") enableSuite[GlutenSortOrderExpressionsSuite].exclude("SortPrefix") enableSuite[GlutenStringExpressionsSuite] - .exclude("concat") .exclude("StringComparison") .exclude("Substring") .exclude("string substring_index function") @@ -766,24 +764,14 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("base64/unbase64 for string") .exclude("encode/decode for string") .exclude("Levenshtein distance") - .exclude("soundex unit test") - .exclude("replace") .exclude("overlay for string") .exclude("overlay for byte array") .exclude("translate") - .exclude("FORMAT") - .exclude("SPARK-22603: FormatString should not generate codes beyond 64KB") - .exclude("INSTR") .exclude("LOCATE") - .exclude("LPAD/RPAD") .exclude("REPEAT") .exclude("length for string / binary") - .exclude("format_number / FormatNumber") - .exclude("ToNumber: positive tests") - .exclude("ToNumber: negative tests (the input string does not match the format string)") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") - .exclude("Sentences") enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite] enableSuite[GlutenDataSourceV2SQLSessionCatalogSuite] enableSuite[GlutenDataSourceV2SQLSuiteV1Filter] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 07af1fa845ca..38ed2c53463b 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -457,7 +457,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string regex_replace / regex_extract") .exclude("string overlay function") .exclude("binary overlay function") - .exclude("string / binary substring function") .exclude("string parse_url function") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") @@ -756,7 +755,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK - 34814: LikeSimplification should handleNULL") enableSuite[GlutenSortOrderExpressionsSuite].exclude("SortPrefix") enableSuite[GlutenStringExpressionsSuite] - .exclude("concat") .exclude("StringComparison") .exclude("Substring") .exclude("string substring_index function") @@ -766,24 +764,14 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("base64/unbase64 for string") .exclude("encode/decode for string") .exclude("Levenshtein distance") - .exclude("soundex unit test") - .exclude("replace") .exclude("overlay for string") .exclude("overlay for byte array") .exclude("translate") - .exclude("FORMAT") - .exclude("SPARK-22603: FormatString should not generate codes beyond 64KB") - .exclude("INSTR") .exclude("LOCATE") - .exclude("LPAD/RPAD") .exclude("REPEAT") .exclude("length for string / binary") - .exclude("format_number / FormatNumber") - .exclude("ToNumber: positive tests") - .exclude("ToNumber: negative tests (the input string does not match the format string)") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") - .exclude("Sentences") enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite] enableSuite[GlutenDataSourceV2SQLSessionCatalogSuite] enableSuite[GlutenDataSourceV2SQLSuiteV1Filter] From 774c66830ba813a5c6231cb1dd504cdd0c862e75 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:59:48 +0800 Subject: [PATCH 340/402] [GLUTEN-6124][CH]Fix json output diff (#6125) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #6124) How was this patch tested? TEST BY UT --- .../execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala | 5 +++++ cpp-ch/local-engine/Common/CHUtil.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 5040153320fc..118f8418609d 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2048,10 +2048,15 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr """ |select to_json(struct(cast(id as string), id, 1.1, 1.1f, 1.1d)) from range(3) |""".stripMargin + val sql1 = + """ + | select to_json(named_struct('name', concat('/val/', id))) from range(3) + |""".stripMargin // cast('nan' as double) output 'NaN' in Spark, 'nan' in CH // cast('inf' as double) output 'Infinity' in Spark, 'inf' in CH // ignore them temporarily runQueryAndCompare(sql)(checkGlutenOperatorMatch[ProjectExecTransformer]) + runQueryAndCompare(sql1)(checkGlutenOperatorMatch[ProjectExecTransformer]) } test("GLUTEN-3501: test json output format with struct contains null value") { diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 588cc1cb2599..148e78bfbc79 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -672,7 +672,6 @@ void BackendInitializerUtil::initSettings(std::map & b LOG_DEBUG(&Poco::Logger::get("CHUtil"), "Set settings key:{} value:{}", key, value); } } - /// Finally apply some fixed kvs to settings. settings.set("join_use_nulls", true); settings.set("input_format_orc_allow_missing_columns", true); @@ -694,6 +693,7 @@ void BackendInitializerUtil::initSettings(std::map & b settings.set("output_format_json_quote_64bit_integers", false); settings.set("output_format_json_quote_denormals", true); settings.set("output_format_json_skip_null_value_in_named_tuples", true); + settings.set("output_format_json_escape_forward_slashes", false); settings.set("function_json_value_return_type_allow_complex", true); settings.set("function_json_value_return_type_allow_nullable", true); settings.set("precise_float_parsing", true); From 10a663c2b86c73490cdaee1d94177cb485c9fe31 Mon Sep 17 00:00:00 2001 From: KevinyhZou <37431499+KevinyhZou@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:02:00 +0800 Subject: [PATCH 341/402] [GLUTEN-6156][CH]Fix least diff (#6155) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #6156) How was this patch tested? test by ut --- ...enClickHouseTPCHSaltNullParquetSuite.scala | 4 +- .../Functions/FunctionGreatestLeast.h | 77 +++++++++++++++++++ .../Functions/SparkFunctionGreatest.cpp | 47 ++--------- .../Functions/SparkFunctionLeast.cpp | 38 +++++++++ .../Parser/SerializedPlanParser.h | 2 +- 5 files changed, 123 insertions(+), 45 deletions(-) create mode 100644 cpp-ch/local-engine/Functions/FunctionGreatestLeast.h create mode 100644 cpp-ch/local-engine/Functions/SparkFunctionLeast.cpp diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 118f8418609d..188995f11058 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2575,12 +2575,12 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr spark.sql("drop table test_tbl_5096") } - test("GLUTEN-5896: Bug fix greatest diff") { + test("GLUTEN-5896: Bug fix greatest/least diff") { val tbl_create_sql = "create table test_tbl_5896(id bigint, x1 int, x2 int, x3 int) using parquet" val tbl_insert_sql = "insert into test_tbl_5896 values(1, 12, NULL, 13), (2, NULL, NULL, NULL), (3, 11, NULL, NULL), (4, 10, 9, 8)" - val select_sql = "select id, greatest(x1, x2, x3) from test_tbl_5896" + val select_sql = "select id, greatest(x1, x2, x3), least(x1, x2, x3) from test_tbl_5896" spark.sql(tbl_create_sql) spark.sql(tbl_insert_sql) compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) diff --git a/cpp-ch/local-engine/Functions/FunctionGreatestLeast.h b/cpp-ch/local-engine/Functions/FunctionGreatestLeast.h new file mode 100644 index 000000000000..6930c1d75b79 --- /dev/null +++ b/cpp-ch/local-engine/Functions/FunctionGreatestLeast.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} +namespace local_engine +{ +template +class FunctionGreatestestLeast : public DB::FunctionLeastGreatestGeneric +{ +public: + bool useDefaultImplementationForNulls() const override { return false; } + virtual String getName() const = 0; + +private: + DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes & types) const override + { + if (types.empty()) + throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} cannot be called without arguments", getName()); + return makeNullable(getLeastSupertype(types)); + } + + DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows_count) const override + { + size_t num_arguments = arguments.size(); + DB::Columns converted_columns(num_arguments); + for (size_t arg = 0; arg < num_arguments; ++arg) + converted_columns[arg] = castColumn(arguments[arg], result_type)->convertToFullColumnIfConst(); + auto result_column = result_type->createColumn(); + result_column->reserve(input_rows_count); + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + size_t best_arg = 0; + for (size_t arg = 1; arg < num_arguments; ++arg) + { + if constexpr (kind == DB::LeastGreatest::Greatest) + { + auto cmp_result = converted_columns[arg]->compareAt(row_num, row_num, *converted_columns[best_arg], -1); + if (cmp_result > 0) + best_arg = arg; + } + else + { + auto cmp_result = converted_columns[arg]->compareAt(row_num, row_num, *converted_columns[best_arg], 1); + if (cmp_result < 0) + best_arg = arg; + } + } + result_column->insertFrom(*converted_columns[best_arg], row_num); + } + return result_column; + } +}; + +} diff --git a/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp b/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp index 9577d65ec5f7..920fe1b9c9cc 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionGreatest.cpp @@ -14,58 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} -} +#include namespace local_engine { -class SparkFunctionGreatest : public DB::FunctionLeastGreatestGeneric +class SparkFunctionGreatest : public FunctionGreatestestLeast { public: static constexpr auto name = "sparkGreatest"; static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared(); } SparkFunctionGreatest() = default; ~SparkFunctionGreatest() override = default; - bool useDefaultImplementationForNulls() const override { return false; } - -private: - DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes & types) const override - { - if (types.empty()) - throw DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} cannot be called without arguments", name); - return makeNullable(getLeastSupertype(types)); - } - - DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows_count) const override + String getName() const override { - size_t num_arguments = arguments.size(); - DB::Columns converted_columns(num_arguments); - for (size_t arg = 0; arg < num_arguments; ++arg) - converted_columns[arg] = castColumn(arguments[arg], result_type)->convertToFullColumnIfConst(); - auto result_column = result_type->createColumn(); - result_column->reserve(input_rows_count); - for (size_t row_num = 0; row_num < input_rows_count; ++row_num) - { - size_t best_arg = 0; - for (size_t arg = 1; arg < num_arguments; ++arg) - { - auto cmp_result = converted_columns[arg]->compareAt(row_num, row_num, *converted_columns[best_arg], -1); - if (cmp_result > 0) - best_arg = arg; - } - result_column->insertFrom(*converted_columns[best_arg], row_num); - } - return result_column; - } + return name; + } }; REGISTER_FUNCTION(SparkGreatest) diff --git a/cpp-ch/local-engine/Functions/SparkFunctionLeast.cpp b/cpp-ch/local-engine/Functions/SparkFunctionLeast.cpp new file mode 100644 index 000000000000..70aafdf07209 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkFunctionLeast.cpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +namespace local_engine +{ +class SparkFunctionLeast : public FunctionGreatestestLeast +{ +public: + static constexpr auto name = "sparkLeast"; + static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared(); } + SparkFunctionLeast() = default; + ~SparkFunctionLeast() override = default; + String getName() const override + { + return name; + } +}; + +REGISTER_FUNCTION(SparkLeast) +{ + factory.registerFunction(); +} +} diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 6ce92b558b73..184065836e65 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -105,7 +105,7 @@ static const std::map SCALAR_FUNCTIONS {"sign", "sign"}, {"radians", "radians"}, {"greatest", "sparkGreatest"}, - {"least", "least"}, + {"least", "sparkLeast"}, {"shiftleft", "bitShiftLeft"}, {"shiftright", "bitShiftRight"}, {"check_overflow", "checkDecimalOverflowSpark"}, From d91a316c3e981b3b67bf4316d42fce29dff2708d Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 26 Jun 2024 19:30:52 +0800 Subject: [PATCH 342/402] [VL][Minor] Fix udf jni signature mismatch (#6212) * fix udf library path failed to get resolved on yarn-cluster * fix signature * Revert "fix udf library path failed to get resolved on yarn-cluster" This reverts commit 11f774a1cc03ff3c0152dfbcee7aaa450bb6157c. --- .../src/main/java/org/apache/gluten/udf/UdfJniWrapper.java | 4 +--- .../scala/org/apache/spark/sql/expression/UDFResolver.scala | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/backends-velox/src/main/java/org/apache/gluten/udf/UdfJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/udf/UdfJniWrapper.java index 4b609769b2ab..8bfe8bad5c01 100644 --- a/backends-velox/src/main/java/org/apache/gluten/udf/UdfJniWrapper.java +++ b/backends-velox/src/main/java/org/apache/gluten/udf/UdfJniWrapper.java @@ -18,7 +18,5 @@ public class UdfJniWrapper { - public UdfJniWrapper() {} - - public native void getFunctionSignatures(); + public static native void getFunctionSignatures(); } diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index e45e8b6fa6d7..8a549c9b4ea9 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -327,7 +327,7 @@ object UDFResolver extends Logging { case None => Seq.empty case Some(_) => - new UdfJniWrapper().getFunctionSignatures() + UdfJniWrapper.getFunctionSignatures() UDFNames.map { name => From 0800596b5d23c55443fd74fa6461c5264619c6b4 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 26 Jun 2024 20:01:17 +0800 Subject: [PATCH 343/402] [VL] Make jni debug workspace configurable (#6228) --- .../gluten/backendsapi/velox/VeloxListenerApi.scala | 7 +++---- .../org/apache/gluten/vectorized/JniWorkspace.java | 4 ++-- .../main/scala/org/apache/gluten/GlutenConfig.scala | 12 +++++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 41b56804b50b..81f06478cbb6 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -161,10 +161,9 @@ class VeloxListenerApi extends ListenerApi { private def initialize(conf: SparkConf, isDriver: Boolean): Unit = { SparkDirectoryUtil.init(conf) UDFResolver.resolveUdfConf(conf, isDriver = isDriver) - val debugJni = conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_MODE, defaultValue = false) && - conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE, defaultValue = false) - if (debugJni) { - JniWorkspace.enableDebug() + if (conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE, defaultValue = false)) { + val debugDir = conf.get(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE_DIR) + JniWorkspace.enableDebug(debugDir) } val loader = JniWorkspace.getDefault.libLoader diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java index a7c12387a221..810c945d35ab 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java @@ -75,14 +75,14 @@ private static JniWorkspace createDefault() { } } - public static void enableDebug() { + public static void enableDebug(String debugDir) { // Preserve the JNI libraries even after process exits. // This is useful for debugging native code if the debug symbols were embedded in // the libraries. synchronized (DEFAULT_INSTANCE_INIT_LOCK) { if (DEBUG_INSTANCE == null) { final File tempRoot = - Paths.get("/tmp").resolve("gluten-jni-debug-" + UUID.randomUUID()).toFile(); + Paths.get(debugDir).resolve("gluten-jni-debug-" + UUID.randomUUID()).toFile(); try { FileUtils.forceMkdir(tempRoot); } catch (IOException e) { diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index cc2d6ac5fdef..89933cc58a4d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -391,8 +391,7 @@ class GlutenConfig(conf: SQLConf) extends Logging { conf.getConf(COLUMNAR_VELOX_MEMORY_USE_HUGE_PAGES) def debug: Boolean = conf.getConf(DEBUG_ENABLED) - def debugKeepJniWorkspace: Boolean = - conf.getConf(DEBUG_ENABLED) && conf.getConf(DEBUG_KEEP_JNI_WORKSPACE) + def debugKeepJniWorkspace: Boolean = conf.getConf(DEBUG_KEEP_JNI_WORKSPACE) def taskStageId: Int = conf.getConf(BENCHMARK_TASK_STAGEID) def taskPartitionId: Int = conf.getConf(BENCHMARK_TASK_PARTITIONID) def taskId: Long = conf.getConf(BENCHMARK_TASK_TASK_ID) @@ -553,6 +552,7 @@ object GlutenConfig { val GLUTEN_DEBUG_MODE = "spark.gluten.sql.debug" val GLUTEN_DEBUG_KEEP_JNI_WORKSPACE = "spark.gluten.sql.debug.keepJniWorkspace" + val GLUTEN_DEBUG_KEEP_JNI_WORKSPACE_DIR = "spark.gluten.sql.debug.keepJniWorkspaceDir" // Added back to Spark Conf during executor initialization val GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY = "spark.gluten.numTaskSlotsPerExecutor" @@ -1580,11 +1580,17 @@ object GlutenConfig { .createWithDefault(false) val DEBUG_KEEP_JNI_WORKSPACE = - buildConf(GLUTEN_DEBUG_KEEP_JNI_WORKSPACE) + buildStaticConf(GLUTEN_DEBUG_KEEP_JNI_WORKSPACE) .internal() .booleanConf .createWithDefault(false) + val DEBUG_KEEP_JNI_WORKSPACE_DIR = + buildStaticConf(GLUTEN_DEBUG_KEEP_JNI_WORKSPACE_DIR) + .internal() + .stringConf + .createWithDefault("/tmp") + val BENCHMARK_TASK_STAGEID = buildConf("spark.gluten.sql.benchmark_task.stageId") .internal() From dc6abe54a246a1f789bfdd54b2bd7c1f2bf239ab Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 27 Jun 2024 09:02:35 +0800 Subject: [PATCH 344/402] [VL] Link lib jemalloc produced by custom building (#4747) Co-authored-by: BInwei Yang --- cpp/CMake/Buildjemalloc_pic.cmake | 74 +++++++++++++++++ cpp/CMake/Findjemalloc_pic.cmake | 78 +++++------------- cpp/core/CMakeLists.txt | 10 --- cpp/velox/CMakeLists.txt | 11 +++ cpp/velox/memory/VeloxMemoryManager.cc | 7 ++ dev/vcpkg/CONTRIBUTING.md | 6 +- .../ports/jemalloc/fix-configure-ac.patch | 13 +++ dev/vcpkg/ports/jemalloc/portfile.cmake | 79 +++++++++++++++++++ dev/vcpkg/ports/jemalloc/preprocessor.patch | 12 +++ dev/vcpkg/ports/jemalloc/vcpkg.json | 8 ++ docs/get-started/build-guide.md | 2 +- 11 files changed, 226 insertions(+), 74 deletions(-) create mode 100644 cpp/CMake/Buildjemalloc_pic.cmake create mode 100644 dev/vcpkg/ports/jemalloc/fix-configure-ac.patch create mode 100644 dev/vcpkg/ports/jemalloc/portfile.cmake create mode 100644 dev/vcpkg/ports/jemalloc/preprocessor.patch create mode 100644 dev/vcpkg/ports/jemalloc/vcpkg.json diff --git a/cpp/CMake/Buildjemalloc_pic.cmake b/cpp/CMake/Buildjemalloc_pic.cmake new file mode 100644 index 000000000000..7c2316ea9540 --- /dev/null +++ b/cpp/CMake/Buildjemalloc_pic.cmake @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Building Jemalloc +macro(build_jemalloc) + message(STATUS "Building Jemalloc from Source") + + if(DEFINED ENV{GLUTEN_JEMALLOC_URL}) + set(JEMALLOC_SOURCE_URL "$ENV{GLUTEN_JEMALLOC_URL}") + else() + set(JEMALLOC_BUILD_VERSION "5.2.1") + set(JEMALLOC_SOURCE_URL + "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_BUILD_VERSION}/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" + "https://github.com/ursa-labs/thirdparty/releases/download/latest/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" + ) + endif() + + set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-install") + set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib") + set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include") + set(JEMALLOC_STATIC_LIB + "${JEMALLOC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(JEMALLOC_INCLUDE "${JEMALLOC_PREFIX}/include") + set(JEMALLOC_CONFIGURE_ARGS + "AR=${CMAKE_AR}" + "CC=${CMAKE_C_COMPILER}" + "--prefix=${JEMALLOC_PREFIX}" + "--libdir=${JEMALLOC_LIB_DIR}" + "--with-jemalloc-prefix=je_gluten_" + "--with-private-namespace=je_gluten_private_" + "--without-export" + "--disable-shared" + "--disable-cxx" + "--disable-libdl" + # For fixing an issue when loading native lib: cannot allocate memory in + # static TLS block. + "--disable-initial-exec-tls" + "CFLAGS=-fPIC" + "CXXFLAGS=-fPIC") + set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) + ExternalProject_Add( + jemalloc_ep + URL ${JEMALLOC_SOURCE_URL} + PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html + CONFIGURE_COMMAND "./configure" ${JEMALLOC_CONFIGURE_ARGS} + BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" + INSTALL_COMMAND make install) + + file(MAKE_DIRECTORY "${JEMALLOC_INCLUDE_DIR}") + add_library(jemalloc::libjemalloc STATIC IMPORTED) + set_target_properties( + jemalloc::libjemalloc + PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads + IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}") + add_dependencies(jemalloc::libjemalloc jemalloc_ep) +endmacro() diff --git a/cpp/CMake/Findjemalloc_pic.cmake b/cpp/CMake/Findjemalloc_pic.cmake index fae9f0d7ad80..ca7b7d213dfc 100644 --- a/cpp/CMake/Findjemalloc_pic.cmake +++ b/cpp/CMake/Findjemalloc_pic.cmake @@ -17,67 +17,25 @@ # Find Jemalloc macro(find_jemalloc) - # Find the existing Protobuf + # Find the existing jemalloc set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - find_package(jemalloc_pic) - if("${Jemalloc_LIBRARY}" STREQUAL "Jemalloc_LIBRARY-NOTFOUND") - message(FATAL_ERROR "Jemalloc Library Not Found") - endif() - set(PROTOC_BIN ${Jemalloc_PROTOC_EXECUTABLE}) -endmacro() - -# Building Jemalloc -macro(build_jemalloc) - message(STATUS "Building Jemalloc from Source") - - if(DEFINED ENV{GLUTEN_JEMALLOC_URL}) - set(JEMALLOC_SOURCE_URL "$ENV{GLUTEN_JEMALLOC_URL}") + # Find from vcpkg-installed lib path. + find_library( + JEMALLOC_LIBRARY + NAMES jemalloc_pic + PATHS + ${CMAKE_CURRENT_BINARY_DIR}/../../../dev/vcpkg/vcpkg_installed/x64-linux-avx/lib/ + NO_DEFAULT_PATH) + if("${JEMALLOC_LIBRARY}" STREQUAL "JEMALLOC_LIBRARY-NOTFOUND") + message(STATUS "Jemalloc Library Not Found.") + set(JEMALLOC_NOT_FOUND TRUE) else() - set(JEMALLOC_BUILD_VERSION "5.2.1") - set(JEMALLOC_SOURCE_URL - "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_BUILD_VERSION}/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" - "https://github.com/ursa-labs/thirdparty/releases/download/latest/jemalloc-${JEMALLOC_BUILD_VERSION}.tar.bz2" - ) + message(STATUS "Found jemalloc: ${JEMALLOC_LIBRARY}") + find_path(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h) + add_library(jemalloc::libjemalloc STATIC IMPORTED) + set_target_properties( + jemalloc::libjemalloc + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}" + IMPORTED_LOCATION "${JEMALLOC_LIBRARY}") endif() - - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-install") - set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib") - set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include") - set(JEMALLOC_STATIC_LIB - "${JEMALLOC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}jemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(JEMALLOC_INCLUDE "${JEMALLOC_PREFIX}/include") - set(JEMALLOC_CONFIGURE_ARGS - "AR=${CMAKE_AR}" - "CC=${CMAKE_C_COMPILER}" - "--prefix=${JEMALLOC_PREFIX}" - "--libdir=${JEMALLOC_LIB_DIR}" - "--with-jemalloc-prefix=je_gluten_" - "--with-private-namespace=je_gluten_private_" - "--without-export" - "--disable-shared" - "--disable-cxx" - "--disable-libdl" - "--disable-initial-exec-tls" - "CFLAGS=-fPIC" - "CXXFLAGS=-fPIC") - set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) - ExternalProject_Add( - jemalloc_ep - URL ${JEMALLOC_SOURCE_URL} - PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html - CONFIGURE_COMMAND "./configure" ${JEMALLOC_CONFIGURE_ARGS} - BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND} - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" - INSTALL_COMMAND make install) - - file(MAKE_DIRECTORY "${JEMALLOC_INCLUDE_DIR}") - add_library(jemalloc::libjemalloc STATIC IMPORTED) - set_target_properties( - jemalloc::libjemalloc - PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads - IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}") - add_dependencies(jemalloc::libjemalloc protobuf_ep) endmacro() diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 4d7c30402985..e17d13581105 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -300,16 +300,6 @@ target_include_directories( set_target_properties(gluten PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${root_directory}/releases) -include(Findjemalloc_pic) -# Build Jemalloc -if(BUILD_JEMALLOC) - build_jemalloc(${STATIC_JEMALLOC}) - message(STATUS "Building Jemalloc: ${STATIC_JEMALLOC}") -else() # - find_jemalloc() - message(STATUS "Use existing Jemalloc libraries") -endif() - if(BUILD_TESTS) add_subdirectory(tests) endif() diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index c2d690a7e055..716a5f68a91c 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -576,6 +576,17 @@ find_package(Folly REQUIRED CONFIG) target_include_directories(velox PUBLIC ${GTEST_INCLUDE_DIRS} ${PROTOBUF_INCLUDE}) +if(BUILD_JEMALLOC) + include(Findjemalloc_pic) + find_jemalloc() + if(JEMALLOC_NOT_FOUND) + include(Buildjemalloc_pic) + build_jemalloc() + endif() + add_definitions(-DENABLE_JEMALLOC) + target_link_libraries(velox PUBLIC jemalloc::libjemalloc) +endif() + target_link_libraries(velox PUBLIC gluten) add_velox_dependencies() diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index 733eb4c4bc39..efd165b736be 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -16,6 +16,10 @@ */ #include "VeloxMemoryManager.h" +#ifdef ENABLE_JEMALLOC +#include +#endif + #include "velox/common/memory/MallocAllocator.h" #include "velox/common/memory/MemoryPool.h" #include "velox/exec/MemoryReclaimer.h" @@ -326,6 +330,9 @@ VeloxMemoryManager::~VeloxMemoryManager() { usleep(waitMs * 1000); accumulatedWaitMs += waitMs; } +#ifdef ENABLE_JEMALLOC + je_gluten_malloc_stats_print(NULL, NULL, NULL); +#endif } } // namespace gluten diff --git a/dev/vcpkg/CONTRIBUTING.md b/dev/vcpkg/CONTRIBUTING.md index b725f0b50fc5..719bc91db066 100644 --- a/dev/vcpkg/CONTRIBUTING.md +++ b/dev/vcpkg/CONTRIBUTING.md @@ -13,7 +13,7 @@ Please init vcpkg env first: Vcpkg already maintains a lot of libraries. You can find them by vcpkg cli. -(NOTE: Please always use cli beacause [packages on vcpkg.io](https://vcpkg.io/en/packages.html) is outdate). +(NOTE: Please always use cli because [packages on vcpkg.io](https://vcpkg.io/en/packages.html) is outdate). ``` $ ./.vcpkg/vcpkg search folly @@ -28,7 +28,7 @@ folly[zlib] Support zlib for compression folly[zstd] Support zstd for compression ``` -`[...]` means additional features. Then add depend into [vcpkg.json](./vcpkg.json). +`[...]` means additional features. Then add the dependency into [vcpkg.json](./vcpkg.json). ``` json { @@ -144,7 +144,7 @@ See [vcpkg.json reference](https://learn.microsoft.com/en-us/vcpkg/reference/vcp `portfile.cmake` is a cmake script describing how to build and install the package. A typical portfile has 3 stages: -**Download and perpare source**: +**Download and prepare source**: ``` cmake # Download from Github diff --git a/dev/vcpkg/ports/jemalloc/fix-configure-ac.patch b/dev/vcpkg/ports/jemalloc/fix-configure-ac.patch new file mode 100644 index 000000000000..7799dfb9e80e --- /dev/null +++ b/dev/vcpkg/ports/jemalloc/fix-configure-ac.patch @@ -0,0 +1,13 @@ +diff --git a/configure.ac b/configure.ac +index f6d25f334..3115504e2 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1592,7 +1592,7 @@ fi + [enable_uaf_detection="0"] + ) + if test "x$enable_uaf_detection" = "x1" ; then +- AC_DEFINE([JEMALLOC_UAF_DETECTION], [ ]) ++ AC_DEFINE([JEMALLOC_UAF_DETECTION], [ ], ["enable UAF"]) + fi + AC_SUBST([enable_uaf_detection]) + diff --git a/dev/vcpkg/ports/jemalloc/portfile.cmake b/dev/vcpkg/ports/jemalloc/portfile.cmake new file mode 100644 index 000000000000..6cac12ca3b7c --- /dev/null +++ b/dev/vcpkg/ports/jemalloc/portfile.cmake @@ -0,0 +1,79 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO jemalloc/jemalloc + REF 54eaed1d8b56b1aa528be3bdd1877e59c56fa90c + SHA512 527bfbf5db9a5c2b7b04df4785b6ae9d445cff8cb17298bf3e550c88890d2bd7953642d8efaa417580610508279b527d3a3b9e227d17394fd2013c88cb7ae75a + HEAD_REF master + PATCHES + fix-configure-ac.patch + preprocessor.patch +) +if(VCPKG_TARGET_IS_WINDOWS) + set(opts "ac_cv_search_log=none required" + "--without-private-namespace" + "--with-jemalloc-prefix=je_gluten_" + "--with-private-namespace=je_gluten_private_" + "--without-export" + "--disable-shared" + "--disable-cxx" + "--disable-libdl" + # For fixing an issue when loading native lib: cannot allocate memory in static TLS block. + "--disable-initial-exec-tls" + "CFLAGS=-fPIC" + "CXXFLAGS=-fPIC") +else() + set(opts + "--with-jemalloc-prefix=je_gluten_" + "--with-private-namespace=je_gluten_private_" + "--without-export" + "--disable-shared" + "--disable-cxx" + "--disable-libdl" + # For fixing an issue when loading native lib: cannot allocate memory in static TLS block. + "--disable-initial-exec-tls" + "CFLAGS=-fPIC" + "CXXFLAGS=-fPIC") +endif() + +vcpkg_configure_make( + SOURCE_PATH "${SOURCE_PATH}" + AUTOCONFIG + NO_WRAPPERS + OPTIONS ${opts} +) + +vcpkg_install_make() + +if(VCPKG_TARGET_IS_WINDOWS) + file(COPY "${SOURCE_PATH}/include/msvc_compat/strings.h" DESTINATION "${CURRENT_PACKAGES_DIR}/include/jemalloc/msvc_compat") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/jemalloc/jemalloc.h" "" "\"msvc_compat/strings.h\"") + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + file(COPY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/lib/jemalloc.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/bin") + file(RENAME "${CURRENT_PACKAGES_DIR}/lib/jemalloc.dll" "${CURRENT_PACKAGES_DIR}/bin/jemalloc.dll") + endif() + if(NOT VCPKG_BUILD_TYPE) + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + file(COPY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg/lib/jemalloc.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/debug/bin") + file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/jemalloc.dll" "${CURRENT_PACKAGES_DIR}/debug/bin/jemalloc.dll") + endif() + endif() + if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/jemalloc.pc" "install_suffix=" "install_suffix=_s") + if(NOT VCPKG_BUILD_TYPE) + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/jemalloc.pc" "install_suffix=" "install_suffix=_s") + endif() + endif() +endif() + +vcpkg_fixup_pkgconfig() + +vcpkg_copy_pdbs() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/tools") + +# Handle copyright +file(INSTALL "${SOURCE_PATH}/COPYING" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/dev/vcpkg/ports/jemalloc/preprocessor.patch b/dev/vcpkg/ports/jemalloc/preprocessor.patch new file mode 100644 index 000000000000..6e6e2d1403fb --- /dev/null +++ b/dev/vcpkg/ports/jemalloc/preprocessor.patch @@ -0,0 +1,12 @@ +diff --git a/configure.ac b/configure.ac +index 3115504e2..ffb504b08 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -749,6 +749,7 @@ case "${host}" in + so="dll" + if test "x$je_cv_msvc" = "xyes" ; then + importlib="lib" ++ JE_APPEND_VS(CPPFLAGS, -DJEMALLOC_NO_PRIVATE_NAMESPACE) + DSO_LDFLAGS="-LD" + EXTRA_LDFLAGS="-link -DEBUG" + CTARGET='-Fo$@' diff --git a/dev/vcpkg/ports/jemalloc/vcpkg.json b/dev/vcpkg/ports/jemalloc/vcpkg.json new file mode 100644 index 000000000000..007e05b931c9 --- /dev/null +++ b/dev/vcpkg/ports/jemalloc/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "jemalloc", + "version": "5.3.0", + "port-version": 1, + "description": "jemalloc is a general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support", + "homepage": "https://jemalloc.net/", + "license": "BSD-2-Clause" +} diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index 3db2244ba229..b2e4b9560301 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -14,7 +14,7 @@ Please set them via `--`, e.g. `--build_type=Release`. | build_tests | Build gluten cpp tests. | OFF | | build_examples | Build udf example. | OFF | | build_benchmarks | Build gluten cpp benchmarks. | OFF | -| build_jemalloc | Build with jemalloc. | ON | +| build_jemalloc | Build with jemalloc. | OFF | | build_protobuf | Build protobuf lib. | ON | | enable_qat | Enable QAT for shuffle data de/compression. | OFF | | enable_iaa | Enable IAA for shuffle data de/compression. | OFF | From ac227ded59b5e1b7913bcd137bd46b52c7532303 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 27 Jun 2024 09:16:17 +0800 Subject: [PATCH 345/402] [VL] Remove the registry for Velox's prestosql scalar functions (#5202) --- .../functions/RegistrationAllFunctions.cc | 16 ++++++++++------ cpp/velox/substrait/SubstraitParser.cc | 12 +----------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index b827690d1cdf..638dbcccff0c 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -26,7 +26,6 @@ #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" #include "velox/functions/prestosql/registration/RegistrationFunctions.h" #include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" -#include "velox/functions/sparksql/Bitwise.h" #include "velox/functions/sparksql/Hash.h" #include "velox/functions/sparksql/Rand.h" #include "velox/functions/sparksql/Register.h" @@ -35,6 +34,14 @@ using namespace facebook; +namespace facebook::velox::functions { +void registerPrestoVectorFunctions() { + // Presto function. To be removed. + VELOX_REGISTER_VECTOR_FUNCTION(udf_arrays_overlap, "arrays_overlap"); + VELOX_REGISTER_VECTOR_FUNCTION(udf_transform_keys, "transform_keys"); + VELOX_REGISTER_VECTOR_FUNCTION(udf_transform_values, "transform_values"); +} +} // namespace facebook::velox::functions namespace gluten { namespace { void registerFunctionOverwrite() { @@ -67,19 +74,16 @@ void registerFunctionOverwrite() { velox::exec::registerFunctionCallToSpecialForm( kRowConstructorWithAllNull, std::make_unique(kRowConstructorWithAllNull)); - velox::functions::sparksql::registerBitwiseFunctions("spark_"); velox::functions::registerBinaryIntegral({"check_add"}); velox::functions::registerBinaryIntegral({"check_subtract"}); velox::functions::registerBinaryIntegral({"check_multiply"}); velox::functions::registerBinaryIntegral({"check_divide"}); + + velox::functions::registerPrestoVectorFunctions(); } } // namespace void registerAllFunctions() { - // The registration order matters. Spark sql functions are registered after - // presto sql functions to overwrite the registration for same named - // functions. - velox::functions::prestosql::registerAllScalarFunctions(); velox::functions::sparksql::registerFunctions(""); velox::aggregate::prestosql::registerAllAggregateFunctions( "", true /*registerCompanionFunctions*/, false /*onlyPrestoSignatures*/, true /*overwrite*/); diff --git a/cpp/velox/substrait/SubstraitParser.cc b/cpp/velox/substrait/SubstraitParser.cc index 5555ecfef954..b842914ca933 100644 --- a/cpp/velox/substrait/SubstraitParser.cc +++ b/cpp/velox/substrait/SubstraitParser.cc @@ -391,23 +391,13 @@ std::unordered_map SubstraitParser::substraitVeloxFunc {"named_struct", "row_constructor"}, {"bit_or", "bitwise_or_agg"}, {"bit_and", "bitwise_and_agg"}, - {"bitwise_and", "spark_bitwise_and"}, - {"bitwise_not", "spark_bitwise_not"}, - {"bitwise_or", "spark_bitwise_or"}, - {"bitwise_xor", "spark_bitwise_xor"}, - // TODO: the below registry for rand functions can be removed - // after presto function registry is removed. - {"rand", "spark_rand"}, {"murmur3hash", "hash_with_seed"}, {"xxhash64", "xxhash64_with_seed"}, {"modulus", "remainder"}, {"date_format", "format_datetime"}, {"collect_set", "set_agg"}, - {"forall", "all_match"}, - {"exists", "any_match"}, {"negative", "unaryminus"}, - {"get_array_item", "get"}, - {"arrays_zip", "zip"}}; + {"get_array_item", "get"}}; const std::unordered_map SubstraitParser::typeMap_ = { {"bool", "BOOLEAN"}, From 32808dd22a0384d0e0bb5011bf2393710a4d5942 Mon Sep 17 00:00:00 2001 From: Kerwin Zhang Date: Thu, 27 Jun 2024 09:58:25 +0800 Subject: [PATCH 346/402] [CELEBORN] Upgrade celeborn to 0.4.1 to support scala 2.13-based compilation (#6226) --- .github/workflows/velox_docker.yml | 6 +++--- docs/get-started/ClickHouse.md | 12 ++++++------ .../gluten/celeborn/CelebornShuffleManager.java | 8 +++++++- .../shuffle/gluten/celeborn/CelebornUtils.java | 14 ++++++++++++-- pom.xml | 2 +- tools/gluten-it/pom.xml | 4 ++-- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 31796c15bdd5..d110d0a6d223 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -521,7 +521,7 @@ jobs: fail-fast: false matrix: spark: ["spark-3.2"] - celeborn: ["celeborn-0.4.0", "celeborn-0.3.2"] + celeborn: ["celeborn-0.4.1", "celeborn-0.3.2-incubating"] runs-on: ubuntu-20.04 container: ubuntu:22.04 steps: @@ -557,8 +557,8 @@ jobs: fi echo "EXTRA_PROFILE: ${EXTRA_PROFILE}" cd /opt && mkdir -p celeborn && \ - wget https://archive.apache.org/dist/incubator/celeborn/${{ matrix.celeborn }}-incubating/apache-${{ matrix.celeborn }}-incubating-bin.tgz && \ - tar xzf apache-${{ matrix.celeborn }}-incubating-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ + wget https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \ + tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \ bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \ bash -c "echo -e 'celeborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf" && \ diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md index 4352a99e55f9..ab24de7a4fd6 100644 --- a/docs/get-started/ClickHouse.md +++ b/docs/get-started/ClickHouse.md @@ -679,13 +679,13 @@ spark.shuffle.manager=org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleMa quickly start a celeborn cluster ```shell -wget https://archive.apache.org/dist/incubator/celeborn/celeborn-0.3.0-incubating/apache-celeborn-0.3.0-incubating-bin.tgz && \ -tar -zxvf apache-celeborn-0.3.0-incubating-bin.tgz && \ -mv apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf.template apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf && \ -mv apache-celeborn-0.3.0-incubating-bin/conf/log4j2.xml.template apache-celeborn-0.3.0-incubating-bin/conf/log4j2.xml && \ +wget https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz && \ +tar -zxvf apache-celeborn-0.3.2-incubating-bin.tgz && \ +mv apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf.template apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \ +mv apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml.template apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml && \ mkdir /opt/hadoop && chmod 777 /opt/hadoop && \ -echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs /tmp\nceleborn.worker.monitor.disk.enabled false" > apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf && \ -bash apache-celeborn-0.3.0-incubating-bin/sbin/start-master.sh && bash apache-celeborn-0.3.0-incubating-bin/sbin/start-worker.sh +echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs /tmp\nceleborn.worker.monitor.disk.enabled false" > apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \ +bash apache-celeborn-0.3.2-incubating-bin/sbin/start-master.sh && bash apache-celeborn-0.3.2-incubating-bin/sbin/start-worker.sh ``` ### Columnar shuffle mode diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index f454cf00c656..d196691d1b14 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -217,7 +217,13 @@ public boolean unregisterShuffle(int shuffleId) { } } return CelebornUtils.unregisterShuffle( - lifecycleManager, shuffleClient, shuffleIdTracker, shuffleId, appUniqueId, isDriver()); + lifecycleManager, + shuffleClient, + shuffleIdTracker, + shuffleId, + appUniqueId, + throwsFetchFailure, + isDriver()); } @Override diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java index 9dd4e1d1191e..6b4229ad3037 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java @@ -49,11 +49,21 @@ public static boolean unregisterShuffle( Object shuffleIdTracker, int appShuffleId, String appUniqueId, + boolean throwsFetchFailure, boolean isDriver) { try { - // for Celeborn 0.4.0 try { - if (lifecycleManager != null) { + try { + // for Celeborn 0.4.1 + if (lifecycleManager != null) { + Method unregisterAppShuffle = + lifecycleManager + .getClass() + .getMethod("unregisterAppShuffle", int.class, boolean.class); + unregisterAppShuffle.invoke(lifecycleManager, appShuffleId, throwsFetchFailure); + } + } catch (NoSuchMethodException ex) { + // for Celeborn 0.4.0 Method unregisterAppShuffle = lifecycleManager.getClass().getMethod("unregisterAppShuffle", int.class); unregisterAppShuffle.invoke(lifecycleManager, appShuffleId); diff --git a/pom.xml b/pom.xml index 81ce0e5d462a..887839ce5fc0 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ delta-core 2.4.0 24 - 0.3.2-incubating + 0.4.1 0.8.0 15.0.0 15.0.0-gluten diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index 3f1760069792..71db637a8403 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -21,7 +21,7 @@ 3.4.2 2.12 3 - 0.3.0-incubating + 0.3.2-incubating 0.8.0 1.2.0-SNAPSHOT 32.0.1-jre @@ -167,7 +167,7 @@ celeborn-0.4 - 0.4.0-incubating + 0.4.1 From 3a42e8fbd3797390a554839ef10e6f9b073460d6 Mon Sep 17 00:00:00 2001 From: Kerwin Zhang Date: Thu, 27 Jun 2024 11:49:15 +0800 Subject: [PATCH 347/402] [CELEBORN] Add config to control celeborn fallback for CI (#6230) --- .../gluten/celeborn/CelebornShuffleManager.java | 12 +++++++++--- .../main/scala/org/apache/gluten/GlutenConfig.scala | 10 ++++++++++ .../org/apache/gluten/integration/Constants.scala | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index d196691d1b14..63fb0cc1b9bd 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -16,6 +16,7 @@ */ package org.apache.spark.shuffle.gluten.celeborn; +import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; import org.apache.gluten.exception.GlutenException; @@ -194,9 +195,14 @@ public ShuffleHandle registerShuffle( if (dependency instanceof ColumnarShuffleDependency) { if (fallbackPolicyRunner.applyAllFallbackPolicy( lifecycleManager, dependency.partitioner().numPartitions())) { - logger.warn("Fallback to ColumnarShuffleManager!"); - columnarShuffleIds.add(shuffleId); - return columnarShuffleManager().registerShuffle(shuffleId, dependency); + if (GlutenConfig.getConf().enableCelebornFallback()) { + logger.warn("Fallback to ColumnarShuffleManager!"); + columnarShuffleIds.add(shuffleId); + return columnarShuffleManager().registerShuffle(shuffleId, dependency); + } else { + throw new GlutenException( + "The Celeborn service(Master: " + celebornConf.masterHost() + ") is unavailable"); + } } else { return registerCelebornShuffleHandle(shuffleId, dependency); } diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 89933cc58a4d..58b99a7f3064 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -447,6 +447,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) def enableHiveFileFormatWriter: Boolean = conf.getConf(NATIVE_HIVEFILEFORMAT_WRITER_ENABLED) + + def enableCelebornFallback: Boolean = conf.getConf(CELEBORN_FALLBACK_ENABLED) } object GlutenConfig { @@ -2049,4 +2051,12 @@ object GlutenConfig { .doubleConf .checkValue(v => v >= 0 && v <= 1, "offheap sizing memory fraction must between [0, 1]") .createWithDefault(0.6) + + val CELEBORN_FALLBACK_ENABLED = + buildStaticConf("spark.gluten.sql.columnar.shuffle.celeborn.fallback.enabled") + .internal() + .doc("If enabled, fall back to ColumnarShuffleManager when celeborn service is unavailable." + + "Otherwise, throw an exception.") + .booleanConf + .createWithDefault(true) } diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala index 50766f3a91d1..e680ce9d5dda 100644 --- a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala +++ b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Constants.scala @@ -44,6 +44,7 @@ object Constants { val VELOX_WITH_CELEBORN_CONF: SparkConf = new SparkConf(false) .set("spark.gluten.sql.columnar.forceShuffledHashJoin", "true") + .set("spark.gluten.sql.columnar.shuffle.celeborn.fallback.enabled", "false") .set("spark.sql.parquet.enableVectorizedReader", "true") .set("spark.plugins", "org.apache.gluten.GlutenPlugin") .set( From b65ecced292b9defdc10d6d5e3a46f43c28fd84c Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 27 Jun 2024 14:06:26 +0800 Subject: [PATCH 348/402] [VL] Remove useless function registering code (#6245) --- .../functions/RegistrationAllFunctions.cc | 7 ------- .../gluten/expression/ExpressionConverter.scala | 16 ++++++++-------- .../gluten/expression/ExpressionNames.scala | 8 ++++---- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index 638dbcccff0c..6b6564fa4aa3 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -52,9 +52,6 @@ void registerFunctionOverwrite() { velox::registerFunction({"round"}); velox::registerFunction({"round"}); velox::registerFunction({"round"}); - // TODO: the below rand function registry can be removed after presto function registry is removed. - velox::registerFunction>({"spark_rand"}); - velox::registerFunction>({"spark_rand"}); auto kRowConstructorWithNull = RowConstructorWithNullCallToSpecialForm::kRowConstructorWithNull; velox::exec::registerVectorFunction( @@ -74,10 +71,6 @@ void registerFunctionOverwrite() { velox::exec::registerFunctionCallToSpecialForm( kRowConstructorWithAllNull, std::make_unique(kRowConstructorWithAllNull)); - velox::functions::registerBinaryIntegral({"check_add"}); - velox::functions::registerBinaryIntegral({"check_subtract"}); - velox::functions::registerBinaryIntegral({"check_multiply"}); - velox::functions::registerBinaryIntegral({"check_divide"}); velox::functions::registerPrestoVectorFunctions(); } diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index da5625cd45e5..d5222cfc6350 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -564,7 +564,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), tryEval, - ExpressionNames.CHECK_ADD + ExpressionNames.CHECKED_ADD ) case tryEval @ TryEval(a: Subtract) => BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( @@ -572,7 +572,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), tryEval, - ExpressionNames.CHECK_SUBTRACT + ExpressionNames.CHECKED_SUBTRACT ) case tryEval @ TryEval(a: Divide) => BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( @@ -580,7 +580,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), tryEval, - ExpressionNames.CHECK_DIVIDE + ExpressionNames.CHECKED_DIVIDE ) case tryEval @ TryEval(a: Multiply) => BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( @@ -588,7 +588,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), tryEval, - ExpressionNames.CHECK_MULTIPLY + ExpressionNames.CHECKED_MULTIPLY ) case a: Add => BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( @@ -596,7 +596,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), a, - ExpressionNames.CHECK_ADD + ExpressionNames.CHECKED_ADD ) case a: Subtract => BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( @@ -604,7 +604,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), a, - ExpressionNames.CHECK_SUBTRACT + ExpressionNames.CHECKED_SUBTRACT ) case a: Multiply => BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( @@ -612,7 +612,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), a, - ExpressionNames.CHECK_MULTIPLY + ExpressionNames.CHECKED_MULTIPLY ) case a: Divide => BackendsApiManager.getSparkPlanExecApiInstance.genArithmeticTransformer( @@ -620,7 +620,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(a.left, attributeSeq, expressionsMap), replaceWithExpressionTransformerInternal(a.right, attributeSeq, expressionsMap), a, - ExpressionNames.CHECK_DIVIDE + ExpressionNames.CHECKED_DIVIDE ) case tryEval: TryEval => // This is a placeholder to handle try_eval(other expressions). diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 2be3fad9d39d..8317e28b58bb 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -83,10 +83,10 @@ object ExpressionNames { final val IS_NAN = "isnan" final val NANVL = "nanvl" final val TRY_EVAL = "try" - final val CHECK_ADD = "check_add" - final val CHECK_SUBTRACT = "check_subtract" - final val CHECK_DIVIDE = "check_divide" - final val CHECK_MULTIPLY = "check_multiply" + final val CHECKED_ADD = "checked_add" + final val CHECKED_SUBTRACT = "checked_subtract" + final val CHECKED_DIVIDE = "checked_divide" + final val CHECKED_MULTIPLY = "checked_multiply" // SparkSQL String functions final val ASCII = "ascii" From 51b1901a797c8dc43f11793fd9b679d2477a69aa Mon Sep 17 00:00:00 2001 From: exmy Date: Thu, 27 Jun 2024 14:34:02 +0800 Subject: [PATCH 349/402] [GLUTEN-6235][CH] Fix crash on ExpandTransform::work() (#6238) [CH] Fix crash on ExpandTransform::work() --- .../GlutenClickHouseHiveTableSuite.scala | 25 +++++++++++++++++++ .../local-engine/Operator/ExpandTransform.cpp | 2 +- .../execution/BasicScanExecTransformer.scala | 19 +------------- .../execution/AbstractHiveTableScanExec.scala | 2 +- .../execution/AbstractHiveTableScanExec.scala | 2 +- .../execution/AbstractHiveTableScanExec.scala | 2 +- .../execution/AbstractHiveTableScanExec.scala | 2 +- 7 files changed, 31 insertions(+), 23 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala index 9b52f6a8cb53..4e190c087920 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseHiveTableSuite.scala @@ -1252,4 +1252,29 @@ class GlutenClickHouseHiveTableSuite } spark.sql("drop table test_tbl_3452") } + + test("GLUTEN-6235: Fix crash on ExpandTransform::work()") { + val tbl = "test_tbl_6235" + sql(s"drop table if exists $tbl") + val createSql = + s""" + |create table $tbl + |stored as textfile + |as select 1 as a1, 2 as a2, 3 as a3, 4 as a4, 5 as a5, 6 as a6, 7 as a7, 8 as a8, 9 as a9 + |""".stripMargin + sql(createSql) + val select_sql = + s""" + |select + |a5,a6,a7,a8,a3,a4,a9 + |,count(distinct a2) as a2 + |,count(distinct a1) as a1 + |,count(distinct if(a3=1,a2,null)) as a33 + |,count(distinct if(a4=2,a1,null)) as a43 + |from $tbl + |group by a5,a6,a7,a8,a3,a4,a9 with cube + |""".stripMargin + compareResultsAgainstVanillaSpark(select_sql, true, { _ => }) + sql(s"drop table if exists $tbl") + } } diff --git a/cpp-ch/local-engine/Operator/ExpandTransform.cpp b/cpp-ch/local-engine/Operator/ExpandTransform.cpp index 106c38e2d8c3..f5787163c5a1 100644 --- a/cpp-ch/local-engine/Operator/ExpandTransform.cpp +++ b/cpp-ch/local-engine/Operator/ExpandTransform.cpp @@ -104,7 +104,7 @@ void ExpandTransform::work() if (kind == EXPAND_FIELD_KIND_SELECTION) { - const auto & original_col = original_cols[field.get()]; + const auto & original_col = original_cols.at(field.get()); if (type->isNullable() == original_col->isNullable()) { cols.push_back(original_col); diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index 9d231bbc2891..64071fb14c0c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -110,7 +110,7 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource } override protected def doTransform(context: SubstraitContext): TransformContext = { - val output = filterRedundantField(outputAttributes()) + val output = outputAttributes() val typeNodes = ConverterUtils.collectAttributeTypeNodes(output) val nameList = ConverterUtils.collectAttributeNamesWithoutExprId(output) val columnTypeNodes = output.map { @@ -156,21 +156,4 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource context.nextOperatorId(this.nodeName)) TransformContext(output, output, readNode) } - - private def filterRedundantField(outputs: Seq[Attribute]): Seq[Attribute] = { - var finalOutput: List[Attribute] = List() - val outputList = outputs.toArray - for (i <- outputList.indices) { - var dup = false - for (j <- 0 until i) { - if (outputList(i).name == outputList(j).name) { - dup = true - } - } - if (!dup) { - finalOutput = finalOutput :+ outputList(i) - } - } - finalOutput - } } diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala index 95106b4edba1..46b59ac306c2 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala @@ -75,7 +75,7 @@ abstract private[hive] class AbstractHiveTableScanExec( override val output: Seq[Attribute] = { // Retrieve the original attributes based on expression ID so that capitalization matches. - requestedAttributes.map(originalAttributes) + requestedAttributes.map(originalAttributes).distinct } // Bind all partition key attribute references in the partition pruning predicate for later diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala index 78f5ff7f1be1..dd095f0ff247 100644 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala @@ -75,7 +75,7 @@ abstract private[hive] class AbstractHiveTableScanExec( override val output: Seq[Attribute] = { // Retrieve the original attributes based on expression ID so that capitalization matches. - requestedAttributes.map(originalAttributes) + requestedAttributes.map(originalAttributes).distinct } // Bind all partition key attribute references in the partition pruning predicate for later diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala index 77f15ac57087..87aba00b0f59 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala @@ -77,7 +77,7 @@ abstract private[hive] class AbstractHiveTableScanExec( override val output: Seq[Attribute] = { // Retrieve the original attributes based on expression ID so that capitalization matches. - requestedAttributes.map(originalAttributes) + requestedAttributes.map(originalAttributes).distinct } // Bind all partition key attribute references in the partition pruning predicate for later diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala index 77f15ac57087..87aba00b0f59 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/hive/execution/AbstractHiveTableScanExec.scala @@ -77,7 +77,7 @@ abstract private[hive] class AbstractHiveTableScanExec( override val output: Seq[Attribute] = { // Retrieve the original attributes based on expression ID so that capitalization matches. - requestedAttributes.map(originalAttributes) + requestedAttributes.map(originalAttributes).distinct } // Bind all partition key attribute references in the partition pruning predicate for later From e71a0c414ecd2595b166a03ee381845f9977302c Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Thu, 27 Jun 2024 15:06:11 +0800 Subject: [PATCH 350/402] [CH] Support use dynamic disk path #6232 What changes were proposed in this pull request? Support use dynamic disk path spark.gluten.sql.columnar.backend.ch.runtime_config.use_current_directory_as_tmp=true disk.metadata_path and cache.path are automatically mapped to the current directory spark.gluten.sql.columnar.backend.ch.runtime_config.reuse_disk_cache=false Add the current pid number to disk.metadata_path and cache.path How was this patch tested? unit tests --- cpp-ch/local-engine/Common/CHUtil.cpp | 75 +++++++++++++++++++++++++++ cpp-ch/local-engine/Common/CHUtil.h | 4 ++ 2 files changed, 79 insertions(+) diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 148e78bfbc79..76c71ce752d6 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -17,6 +17,7 @@ #include "CHUtil.h" #include +#include #include #include #include @@ -527,6 +528,50 @@ std::map BackendInitializerUtil::getBackendConfMap(con return ch_backend_conf; } +std::vector BackendInitializerUtil::wrapDiskPathConfig( + const String & path_prefix, + const String & path_suffix, + Poco::Util::AbstractConfiguration & config) +{ + std::vector changed_paths; + if (path_prefix.empty() && path_suffix.empty()) + return changed_paths; + Poco::Util::AbstractConfiguration::Keys disks; + std::unordered_set disk_types = {"s3", "hdfs_gluten", "cache"}; + config.keys("storage_configuration.disks", disks); + + std::ranges::for_each( + disks, + [&](const auto & disk_name) + { + String disk_prefix = "storage_configuration.disks." + disk_name; + String disk_type = config.getString(disk_prefix + ".type", ""); + if (!disk_types.contains(disk_type)) + return; + if (disk_type == "cache") + { + String path = config.getString(disk_prefix + ".path", ""); + if (!path.empty()) + { + String final_path = path_prefix + path + path_suffix; + config.setString(disk_prefix + ".path", final_path); + changed_paths.emplace_back(final_path); + } + } + else if (disk_type == "s3" || disk_type == "hdfs_gluten") + { + String metadata_path = config.getString(disk_prefix + ".metadata_path", ""); + if (!metadata_path.empty()) + { + String final_path = path_prefix + metadata_path + path_suffix; + config.setString(disk_prefix + ".metadata_path", final_path); + changed_paths.emplace_back(final_path); + } + } + }); + return changed_paths; +} + DB::Context::ConfigurationPtr BackendInitializerUtil::initConfig(std::map & backend_conf_map) { DB::Context::ConfigurationPtr config; @@ -566,6 +611,25 @@ DB::Context::ConfigurationPtr BackendInitializerUtil::initConfig(std::mapsetString(CH_TASK_MEMORY, backend_conf_map.at(GLUTEN_TASK_OFFHEAP)); } + const bool use_current_directory_as_tmp = config->getBool("use_current_directory_as_tmp", false); + char buffer[PATH_MAX]; + if (use_current_directory_as_tmp && getcwd(buffer, sizeof(buffer)) != nullptr) + { + wrapDiskPathConfig(String(buffer), "", *config); + } + + const bool reuse_disk_cache = config->getBool("reuse_disk_cache", true); + + if (!reuse_disk_cache) + { + String pid = std::to_string(static_cast(getpid())); + auto path_need_clean = wrapDiskPathConfig("", "/" + pid, *config); + std::lock_guard lock(BackendFinalizerUtil::paths_mutex); + BackendFinalizerUtil::paths_need_to_clean.insert( + BackendFinalizerUtil::paths_need_to_clean.end(), + path_need_clean.begin(), + path_need_clean.end()); + } return config; } @@ -936,12 +1000,23 @@ void BackendFinalizerUtil::finalizeGlobally() global_context.reset(); shared_context.reset(); } + std::lock_guard lock(paths_mutex); + std::ranges::for_each(paths_need_to_clean, [](const auto & path) + { + if (fs::exists(path)) + fs::remove_all(path); + }); + paths_need_to_clean.clear(); } void BackendFinalizerUtil::finalizeSessionally() { } +std::vector BackendFinalizerUtil::paths_need_to_clean; + +std::mutex BackendFinalizerUtil::paths_mutex; + Int64 DateTimeUtil::currentTimeMillis() { return timeInMilliseconds(std::chrono::system_clock::now()); diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 0321d410a7d5..1198cfa2195d 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -196,6 +196,7 @@ class BackendInitializerUtil static void registerAllFactories(); static void applyGlobalConfigAndSettings(DB::Context::ConfigurationPtr, DB::Settings &); static void updateNewSettings(const DB::ContextMutablePtr &, const DB::Settings &); + static std::vector wrapDiskPathConfig(const String & path_prefix, const String & path_suffix, Poco::Util::AbstractConfiguration & config); static std::map getBackendConfMap(const std::string & plan); @@ -212,6 +213,9 @@ class BackendFinalizerUtil /// Release session level resources like StorageJoinBuilder. Invoked every time executor/driver shutdown. static void finalizeSessionally(); + + static std::vector paths_need_to_clean; + static std::mutex paths_mutex; }; // Ignore memory track, memory should free before IgnoreMemoryTracker deconstruction From 7bf6cd41062c0c6c60a9b17a81674828e02d6a6b Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:15:15 +0800 Subject: [PATCH 351/402] [VL] Daily Update Velox Version (2024_06_27) (#6242) 43cb72a1e by Masha Basmanova, Add support for minus(timestamp with tz, timestamp with tz) Presto function (#10327) 51f86b176 by Masha Basmanova, Add support for TIMESTAMP WITH TIME ZONE inputs to least/greatest Presto functions (#10328) 9857a2eb5 by zhli1142015, Fix NaN handling in Spark In function (#10259) 5a150cec1 by wypb, Fix AbfsReadFile::Impl::preadv to return the length of read. (#10320) 31800f52d by Masha Basmanova, Add support for VARBINARY input to from_base64 Presto function (#10325) 7add4bf64 by Jialiang Tan, Add stats reporter to op test base (#10296) c03401967 by lingbin, Simplify assertion in AllocationTraits (#10322) 6af663c42 by xiaoxmeng, Fix flaky async data cache shutdown test (#10318) 1ae622476 by Yoav Helfman, Fix IOStats for Nimble (#10216) 2a140f9d6 by Jialiang Tan, Shorten AggregationTest.maxSpillBytes from 2m to 2s (#10317) 05222475a by Kevin Wilfong, Ensure the shared HashStringAllocator isn't mutated by Aggregations during spilling (#10309) 622d31ac5 by Kevin Wilfong, Fix race conditions in AsyncDataCache AccessStats (#10312) 136d66be0 by Kevin Wilfong, Fix race condition in MemoryArbitrationFuzzer (#10314) a4e0b6a1f by Kevin Wilfong, Ignore TSAN errors in WindowFuzzer (#10315) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index a96719dc10fc..237757d818d0 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_26 +VELOX_BRANCH=2024_06_27 VELOX_HOME="" #Set on run gluten on HDFS From f837f060f60bb8e5ad173ce92d2a85c294da5d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Thu, 27 Jun 2024 19:52:06 +0800 Subject: [PATCH 352/402] [UT] Remove isVeloxBackendLoaded usage from file metadata UT (#6249) --- .../datasources/GlutenFileMetadataStructSuite.scala | 8 ++++---- .../datasources/GlutenFileMetadataStructSuite.scala | 8 ++++---- .../datasources/GlutenFileMetadataStructSuite.scala | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index efa0fbae062b..ed347d024c1c 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} -import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait @@ -108,7 +108,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS METADATA_FILE_MODIFICATION_TIME, "age") dfWithMetadata.collect - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) } else { checkOperatorMatch[FileSourceScanExec](dfWithMetadata) @@ -133,7 +133,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) val ret = filterDF.collect assert(ret.size == 1) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) @@ -149,7 +149,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS Row(f1(METADATA_FILE_PATH)) ) ) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index b3b9ea7393c3..6e47a94e3c13 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} -import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait @@ -109,7 +109,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS METADATA_FILE_MODIFICATION_TIME, "age") dfWithMetadata.collect - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) } else { checkOperatorMatch[FileSourceScanExec](dfWithMetadata) @@ -134,7 +134,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) val ret = filterDF.collect assert(ret.size == 1) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) @@ -150,7 +150,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS Row(f1(METADATA_FILE_PATH)) ) ) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala index efa0fbae062b..ed347d024c1c 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenFileMetadataStructSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.datasources +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.{FileSourceScanExecTransformer, FilterExecTransformer} -import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{Column, DataFrame, Row} import org.apache.spark.sql.GlutenSQLTestsBaseTrait @@ -108,7 +108,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS METADATA_FILE_MODIFICATION_TIME, "age") dfWithMetadata.collect - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](dfWithMetadata) } else { checkOperatorMatch[FileSourceScanExec](dfWithMetadata) @@ -133,7 +133,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS .where(Column(METADATA_FILE_NAME) === f0((METADATA_FILE_NAME))) val ret = filterDF.collect assert(ret.size == 1) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) @@ -149,7 +149,7 @@ class GlutenFileMetadataStructSuite extends FileMetadataStructSuite with GlutenS Row(f1(METADATA_FILE_PATH)) ) ) - if (BackendTestUtils.isVeloxBackendLoaded()) { + if (BackendsApiManager.getSettings.supportNativeMetadataColumns()) { checkOperatorMatch[FileSourceScanExecTransformer](filterDF) } else { checkOperatorMatch[FileSourceScanExec](filterDF) From eaace6c16f5355ae30872bfd947ca36b550cd497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Thu, 27 Jun 2024 19:56:42 +0800 Subject: [PATCH 353/402] [CORE] Log unknown fallback reason (#6237) --- .../org/apache/spark/sql/execution/GlutenFallbackReporter.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala index 00b0248aee77..721a30eb4f40 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala @@ -89,6 +89,7 @@ case class GlutenFallbackReporter(glutenConfig: GlutenConfig, spark: SparkSessio logicalPlan.setTagValue(FALLBACK_REASON_TAG, newReason) } case TRANSFORM_UNSUPPORTED(_, _) => + logFallbackReason(validationLogLevel, p.nodeName, "unknown reason") case _ => throw new IllegalStateException("Unreachable code") } From 22dc4fdcb5197e7c4a7fdfd768f5abf7a85b354f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Thu, 27 Jun 2024 20:34:23 +0800 Subject: [PATCH 354/402] [GLUTEN-2790][CH] Fix diff between ch char and spark chr (#6236) [CH] Fix diff between ch char and spark chr --- .../Parser/SerializedPlanParser.h | 1 - .../Parser/scalar_function_parser/chr.cpp | 71 +++++++++++++++++++ .../clickhouse/ClickHouseTestSettings.scala | 1 - .../clickhouse/ClickHouseTestSettings.scala | 1 - .../clickhouse/ClickHouseTestSettings.scala | 1 - .../clickhouse/ClickHouseTestSettings.scala | 1 - 6 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/chr.cpp diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 184065836e65..1785f64ee17c 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -133,7 +133,6 @@ static const std::map SCALAR_FUNCTIONS {"replace", "replaceAll"}, {"regexp_replace", "replaceRegexpAll"}, {"regexp_extract_all", "regexpExtractAllSpark"}, - {"chr", "char"}, {"rlike", "match"}, {"ascii", "ascii"}, {"split", "splitByRegexp"}, diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/chr.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/chr.cpp new file mode 100644 index 000000000000..d168e63d11dc --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/chr.cpp @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class FunctionParserChr : public FunctionParser +{ +public: + explicit FunctionParserChr(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } + ~FunctionParserChr() override = default; + static constexpr auto name = "chr"; + String getName() const override { return name; } + + const ActionsDAG::Node * parse( + const substrait::Expression_ScalarFunction & substrait_func, + ActionsDAGPtr & actions_dag) const override + { + auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); + if (parsed_args.size() != 1) + throw Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires two or three arguments", getName()); + + /* + parse chr(number) as if(number < 0, '', convertCharset(char(0, number), 'unicode', 'utf-8')) + */ + const auto & num_arg = parsed_args[0]; + const auto * const_zero_node = addColumnToActionsDAG(actions_dag, std::make_shared(), 0); + const auto * const_empty_node = addColumnToActionsDAG(actions_dag, std::make_shared(), ""); + const auto * const_four_node = addColumnToActionsDAG(actions_dag, std::make_shared(), 4); + const auto * const_unicode_node = addColumnToActionsDAG(actions_dag, std::make_shared(), "unicode"); + const auto * const_utf8_node = addColumnToActionsDAG(actions_dag, std::make_shared(), "utf-8"); + + const auto * less_node = toFunctionNode(actions_dag, "less", {num_arg, const_zero_node}); + + const auto * char_node = toFunctionNode(actions_dag, "char", {const_zero_node, num_arg}); + const auto * convert_charset_node = toFunctionNode(actions_dag, "convertCharset", {char_node, const_unicode_node, const_utf8_node}); + + const auto * if_node = toFunctionNode(actions_dag, "if", {less_node, const_empty_node, convert_charset_node}); + const auto * result_node = convertNodeTypeIfNeeded(substrait_func, if_node, actions_dag); + return result_node; + } +}; + +static FunctionParserRegister register_chr; +} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index d12a40b764f8..3048c3f9cab5 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -897,7 +897,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("Substring") .exclude("string substring_index function") .exclude("ascii for string") - .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") .exclude("overlay for string") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 52e7ebcbda49..769707d4eb5f 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -857,7 +857,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string substring_index function") .exclude("SPARK-40213: ascii for Latin-1 Supplement characters") .exclude("ascii for string") - .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") .exclude("overlay for string") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 38ed2c53463b..268f22fe6981 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -760,7 +760,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string substring_index function") .exclude("SPARK-40213: ascii for Latin-1 Supplement characters") .exclude("ascii for string") - .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") .exclude("Levenshtein distance") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 38ed2c53463b..268f22fe6981 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -760,7 +760,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string substring_index function") .exclude("SPARK-40213: ascii for Latin-1 Supplement characters") .exclude("ascii for string") - .exclude("string for ascii") .exclude("base64/unbase64 for string") .exclude("encode/decode for string") .exclude("Levenshtein distance") From b6776b6960fe4ea0cc2a524d5c254ad64e87a508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Fri, 28 Jun 2024 08:25:28 +0800 Subject: [PATCH 355/402] [CORE] Rename isTransformable API to maybeTransformable (#6233) --- .../extension/columnar/OffloadSingleNode.scala | 11 ++++------- .../extension/columnar/TransformHintRule.scala | 16 +++++----------- .../rewrite/RewriteSparkPlanRulesManager.scala | 2 +- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 8cd2a5fb67bd..75da28e30d39 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -213,18 +213,15 @@ case class OffloadFilter() extends OffloadSingleNode with LogLevelUtil { // Push down the left conditions in Filter into FileSourceScan. val newChild: SparkPlan = filter.child match { case scan @ (_: FileSourceScanExec | _: BatchScanExec) => - if (TransformHints.isTransformable(scan)) { + if (TransformHints.maybeTransformable(scan)) { val newScan = FilterHandler.pushFilterToScan(filter.condition, scan) newScan match { case ts: TransformSupport if ts.doValidate().isValid => ts - // TODO remove the call - case _ => replace.doReplace(scan) + case _ => scan } - } else { - replace.doReplace(scan) - } - case _ => replace.doReplace(filter.child) + } else scan + case _ => filter.child } logDebug(s"Columnar Processing for ${filter.getClass} is currently supported.") BackendsApiManager.getSparkPlanExecApiInstance diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index d32cf2d22eb4..aa7aab759ef8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -79,18 +79,12 @@ object TransformHints { } /** - * NOTE: To be deprecated. Do not create new usages of this method. - * - * Since it's usually not safe to consider a plan "transformable" during validation phase. Another - * validation rule could turn "transformable" to "non-transformable" before implementing the plan - * within Gluten transformers. + * If true, it implies the plan maybe transformable during validation phase but not guaranteed, + * since another validation rule could turn it to "non-transformable" before implementing the plan + * within Gluten transformers. If false, the plan node will be guaranteed fallback to Vanilla plan + * node while being implemented. */ - def isTransformable(plan: SparkPlan): Boolean = { - getHintOption(plan) match { - case None => true - case _ => false - } - } + def maybeTransformable(plan: SparkPlan): Boolean = !isNotTransformable(plan) def tag(plan: SparkPlan, hint: TransformHint): Unit = { val mergedHint = getHintOption(plan) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala index ac663314bead..8706e5618f6b 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala @@ -49,7 +49,7 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] extends Rule[SparkPlan] { private def mayNeedRewrite(plan: SparkPlan): Boolean = { - TransformHints.isTransformable(plan) && { + TransformHints.maybeTransformable(plan) && { plan match { case _: SortExec => true case _: TakeOrderedAndProjectExec => true From b31e863bf4a3081c0adf4120598efe3301f1627b Mon Sep 17 00:00:00 2001 From: Wenzheng Liu Date: Fri, 28 Jun 2024 10:01:36 +0800 Subject: [PATCH 356/402] [GLUTEN-6251][CH] Disable GlutenSortShuffleSuite in clickhouse backend (#6252) --- .../spark/sql/GlutenSQLTestsBaseTrait.scala | 23 -------------- .../spark/sql/GlutenTestsBaseTrait.scala | 30 +++++++++++++++++-- .../spark/sql/GlutenTestsCommonTrait.scala | 23 +------------- .../apache/spark/GlutenSortShuffleSuite.scala | 4 ++- .../apache/spark/GlutenSortShuffleSuite.scala | 4 ++- .../apache/spark/GlutenSortShuffleSuite.scala | 4 ++- .../apache/spark/GlutenSortShuffleSuite.scala | 4 ++- 7 files changed, 41 insertions(+), 51 deletions(-) diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala index 8c55b823a06c..4c06b02a1fb4 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala @@ -20,36 +20,13 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.SparkConf -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, ShuffleQueryStageExec} import org.apache.spark.sql.test.SharedSparkSession -import org.scalactic.source.Position -import org.scalatest.Tag - /** Basic trait for Gluten SQL test cases. */ trait GlutenSQLTestsBaseTrait extends SharedSparkSession with GlutenTestsBaseTrait { - protected def testGluten(testName: String, testTag: Tag*)(testFun: => Any)(implicit - pos: Position): Unit = { - test(GLUTEN_TEST + testName, testTag: _*)(testFun) - } - - protected def ignoreGluten(testName: String, testTag: Tag*)(testFun: => Any)(implicit - pos: Position): Unit = { - super.ignore(GLUTEN_TEST + testName, testTag: _*)(testFun) - } - - override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit - pos: Position): Unit = { - if (shouldRun(testName)) { - super.test(testName, testTags: _*)(testFun) - } else { - super.ignore(testName, testTags: _*)(testFun) - } - } - override def sparkConf: SparkConf = { GlutenSQLTestsBaseTrait.nativeSparkConf(super.sparkConf, warehouse) } diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsBaseTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsBaseTrait.scala index 7c6dcbbee83d..a0ab97306166 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsBaseTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsBaseTrait.scala @@ -18,7 +18,13 @@ package org.apache.spark.sql import org.apache.gluten.utils.BackendTestSettings -trait GlutenTestsBaseTrait { +import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST + +import org.scalactic.source.Position +import org.scalatest.Tag +import org.scalatest.funsuite.AnyFunSuiteLike + +trait GlutenTestsBaseTrait extends AnyFunSuiteLike { protected val rootPath: String = getClass.getResource("/").getPath protected val basePath: String = rootPath + "unit-tests-working-home" @@ -30,7 +36,7 @@ trait GlutenTestsBaseTrait { // list will never be run with no regard to backend test settings. def testNameBlackList: Seq[String] = Seq() - def shouldRun(testName: String): Boolean = { + protected def shouldRun(testName: String): Boolean = { if (testNameBlackList.exists(_.equalsIgnoreCase(GlutenTestConstants.IGNORE_ALL))) { return false } @@ -39,4 +45,24 @@ trait GlutenTestsBaseTrait { } BackendTestSettings.shouldRun(getClass.getCanonicalName, testName) } + + protected def testGluten(testName: String, testTag: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { + test(GLUTEN_TEST + testName, testTag: _*)(testFun) + } + + protected def ignoreGluten(testName: String, testTag: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { + super.ignore(GLUTEN_TEST + testName, testTag: _*)(testFun) + } + + override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { + if (shouldRun(testName)) { + super.test(testName, testTags: _*)(testFun) + } else { + super.ignore(testName, testTags: _*)(testFun) + } + } + } diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsCommonTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsCommonTrait.scala index 06b9fca67bf7..b9ee199eb1af 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsCommonTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsCommonTrait.scala @@ -19,11 +19,9 @@ package org.apache.spark.sql import org.apache.gluten.test.TestStats import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.GlutenTestConstants.GLUTEN_TEST import org.apache.spark.sql.catalyst.expressions._ -import org.scalactic.source.Position -import org.scalatest.{Args, Status, Tag} +import org.scalatest.{Args, Status} trait GlutenTestsCommonTrait extends SparkFunSuite @@ -48,23 +46,4 @@ trait GlutenTestsCommonTrait TestStats.endCase(status.succeeds()); status } - - protected def testGluten(testName: String, testTag: Tag*)(testFun: => Any)(implicit - pos: Position): Unit = { - test(GLUTEN_TEST + testName, testTag: _*)(testFun) - } - - protected def ignoreGluten(testName: String, testTag: Tag*)(testFun: => Any)(implicit - pos: Position): Unit = { - super.ignore(GLUTEN_TEST + testName, testTag: _*)(testFun) - } - - override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit - pos: Position): Unit = { - if (shouldRun(testName)) { - super.test(testName, testTags: _*)(testFun) - } else { - super.ignore(testName, testTags: _*)(testFun) - } - } } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala index 338d7992e38d..70579c886248 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -16,7 +16,9 @@ */ package org.apache.spark -class GlutenSortShuffleSuite extends SortShuffleSuite { +import org.apache.spark.sql.GlutenTestsBaseTrait + +class GlutenSortShuffleSuite extends SortShuffleSuite with GlutenTestsBaseTrait { override def beforeAll(): Unit = { super.beforeAll() conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala index 338d7992e38d..70579c886248 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -16,7 +16,9 @@ */ package org.apache.spark -class GlutenSortShuffleSuite extends SortShuffleSuite { +import org.apache.spark.sql.GlutenTestsBaseTrait + +class GlutenSortShuffleSuite extends SortShuffleSuite with GlutenTestsBaseTrait { override def beforeAll(): Unit = { super.beforeAll() conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala index 338d7992e38d..70579c886248 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -16,7 +16,9 @@ */ package org.apache.spark -class GlutenSortShuffleSuite extends SortShuffleSuite { +import org.apache.spark.sql.GlutenTestsBaseTrait + +class GlutenSortShuffleSuite extends SortShuffleSuite with GlutenTestsBaseTrait { override def beforeAll(): Unit = { super.beforeAll() conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala index 338d7992e38d..70579c886248 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala @@ -16,7 +16,9 @@ */ package org.apache.spark -class GlutenSortShuffleSuite extends SortShuffleSuite { +import org.apache.spark.sql.GlutenTestsBaseTrait + +class GlutenSortShuffleSuite extends SortShuffleSuite with GlutenTestsBaseTrait { override def beforeAll(): Unit = { super.beforeAll() conf.set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") From 4ab76a1493a97ecbc86fb25bacd9f88179ffed1e Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Fri, 28 Jun 2024 10:13:52 +0800 Subject: [PATCH 357/402] [CH] Fix array distinct core dump (#6256) [CH] Fix array distinct core dump Co-authored-by: liuneng1994 --- .../GlutenClickhouseFunctionSuite.scala | 21 +++++++++++++++++++ .../Functions/SparkFunctionArrayDistinct.cpp | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala index 7e46a6989d6f..26e997281221 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala @@ -190,4 +190,25 @@ class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite { )(df => checkFallbackOperators(df, 0)) spark.sql("drop table json_t1") } + + test("Fix arrayDistinct(Array(Nullable(Decimal))) core dump") { + val create_sql = + """ + |create table if not exists test( + | dec array + |) using parquet + |""".stripMargin + val fill_sql = + """ + |insert into test values(array(1, 2, null)), (array(null, 2,3, 5)) + |""".stripMargin + val query_sql = + """ + |select array_distinct(dec) from test; + |""".stripMargin + spark.sql(create_sql) + spark.sql(fill_sql) + compareResultsAgainstVanillaSpark(query_sql, true, { _ => }) + spark.sql("drop table test") + } } diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp b/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp index 89598ff7a1b1..0779346b6e04 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp @@ -314,7 +314,10 @@ void FunctionArrayDistinctSpark::executeHashed( if (!set.find(hash)) { set.insert(hash); - res_data_col.insertFrom(src_data, j); + if (nullable_col) + res_data_col.insertFrom(*nullable_col, j); + else + res_data_col.insertFrom(src_data, j); } } From 2bb813c1c232464383276708ba58602adc5e06c0 Mon Sep 17 00:00:00 2001 From: lgbo Date: Fri, 28 Jun 2024 10:14:21 +0800 Subject: [PATCH 358/402] [GLUTEN-6257][CH] Mismatched headers in broadcast join #6258 What changes were proposed in this pull request? (Please fill in changes proposed in this fix) Fixes: #6257 How was this patch tested? (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) unit tests (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) --- cpp-ch/local-engine/Parser/JoinRelParser.cpp | 23 ++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/cpp-ch/local-engine/Parser/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/JoinRelParser.cpp index 58b156c3cf6e..9a3cc91baaa9 100644 --- a/cpp-ch/local-engine/Parser/JoinRelParser.cpp +++ b/cpp-ch/local-engine/Parser/JoinRelParser.cpp @@ -309,12 +309,27 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const substrait::JoinRel & join, DB::Q // Add a check to find error easily. if (storage_join) { - if(!blocksHaveEqualStructure(right_header_before_convert_step, right->getCurrentDataStream().header)) + bool is_col_names_changed = false; + const auto & current_right_header = right->getCurrentDataStream().header; + if (right_header_before_convert_step.columns() != current_right_header.columns()) + is_col_names_changed = true; + if (!is_col_names_changed) + { + for (size_t i = 0; i < right_header_before_convert_step.columns(); i++) + { + if (right_header_before_convert_step.getByPosition(i).name != current_right_header.getByPosition(i).name) + { + is_col_names_changed = true; + break; + } + } + } + if (is_col_names_changed) { throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "For broadcast join, we must not change the columns name in the right table.\nleft header:{},\nright header: {} -> {}", - left->getCurrentDataStream().header.dumpNames(), - right_header_before_convert_step.dumpNames(), - right->getCurrentDataStream().header.dumpNames()); + left->getCurrentDataStream().header.dumpStructure(), + right_header_before_convert_step.dumpStructure(), + right->getCurrentDataStream().header.dumpStructure()); } } From 1b8be1dee182fe8d6a756725e5a29884613d6d10 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 28 Jun 2024 12:57:31 +0800 Subject: [PATCH 359/402] [VL] Support building arrow CPP and finding installed arrow libs from system (#6229) --- .github/workflows/velox_docker.yml | 10 +- cpp/CMake/ConfigArrow.cmake | 5 +- dev/build_arrow.sh | 92 ++++++++++++++ dev/build_helper_functions.sh | 118 ++++++++++++++++++ dev/builddeps-veloxbe.sh | 29 +++-- .../src/build_arrow_deps_centos8.sh | 45 ------- ep/build-velox/src/build_velox.sh | 39 +----- 7 files changed, 239 insertions(+), 99 deletions(-) create mode 100755 dev/build_arrow.sh create mode 100644 dev/build_helper_functions.sh delete mode 100755 ep/build-velox/src/build_arrow_deps_centos8.sh diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index d110d0a6d223..fd937f6c1d49 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -600,13 +600,11 @@ jobs: - name: Build Gluten Velox third party if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | - cd ep/build-velox/src - ./get_velox.sh source /opt/rh/gcc-toolset-9/enable - ./build_arrow_deps_centos8.sh - ./build_velox.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_test_utils=ON - cd $GITHUB_WORKSPACE/cpp - ./compile.sh --build_velox_backend=ON --build_protobuf=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON + source ./dev/build_arrow.sh + install_arrow_deps + ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \ + --build_examples=ON --build_benchmarks=ON --build_protobuf=ON - uses: actions/upload-artifact@v2 with: name: velox-native-lib-centos-8-${{github.sha}} diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index 8f036be53411..110836347cac 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -33,11 +33,12 @@ function(FIND_ARROW_LIB LIB_NAME) set(ARROW_LIB_FULL_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}) add_library(Arrow::${LIB_NAME} SHARED IMPORTED) + # Firstly find the lib from velox's arrow build path. If not found, try to + # find it from system. find_library( ARROW_LIB_${LIB_NAME} NAMES ${ARROW_LIB_FULL_NAME} - PATHS ${ARROW_LIB_DIR} ${ARROW_LIB64_DIR} - NO_DEFAULT_PATH) + PATHS ${ARROW_LIB_DIR} ${ARROW_LIB64_DIR}) if(NOT ARROW_LIB_${LIB_NAME}) message(FATAL_ERROR "Arrow library Not Found: ${ARROW_LIB_FULL_NAME}") else() diff --git a/dev/build_arrow.sh b/dev/build_arrow.sh new file mode 100755 index 000000000000..a822c4119ea0 --- /dev/null +++ b/dev/build_arrow.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) +source ${CURRENT_DIR}/build_helper_functions.sh +VELOX_ARROW_BUILD_VERSION=15.0.0 +ARROW_PREFIX=$CURRENT_DIR/arrow_ep +# Always uses BUNDLED in case of that thrift is not installed. +THRIFT_SOURCE="BUNDLED" +BUILD_TYPE=Release + +function prepare_arrow_build() { + sudo rm -rf arrow_ep/ + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz arrow_ep + cd arrow_ep/ + patch -p1 < $CURRENT_DIR/../ep/build-velox/src/modify_arrow.patch + patch -p1 < $CURRENT_DIR/../ep/build-velox/src/modify_arrow_dataset_scan_option.patch +} + +function install_arrow_deps { + wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl + pushd openssl + ./config no-shared && make depend && make && sudo make install + popd +} + +function build_arrow_cpp() { + if [ -n "$1" ]; then + BUILD_TYPE=$1 + fi + pushd $ARROW_PREFIX/cpp + + cmake_install \ + -DARROW_PARQUET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_PROTOBUF_USE_SHARED=OFF \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_JEMALLOC=OFF \ + -DARROW_SIMD_LEVEL=NONE \ + -DARROW_RUNTIME_SIMD_LEVEL=NONE \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_TESTING=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DARROW_BUILD_STATIC=ON \ + -DThrift_SOURCE=${THRIFT_SOURCE} + popd +} + +function build_arrow_java() { + ARROW_INSTALL_DIR="${ARROW_PREFIX}/install" + + pushd $ARROW_PREFIX/java + # Because arrow-bom module need the -DprocessAllModules + mvn versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules + + mvn clean install -pl bom,maven/module-info-compiler-maven-plugin,vector -am \ + -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly + + # Arrow C Data Interface CPP libraries + mvn generate-resources -P generate-libs-cdata-all-os -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \ + -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N + + # Arrow JNI Date Interface CPP libraries + export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} + mvn generate-resources -Pgenerate-libs-jni-macos-linux -N -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR \ + -DARROW_GANDIVA=OFF -DARROW_JAVA_JNI_ENABLE_GANDIVA=OFF -DARROW_ORC=OFF -DARROW_JAVA_JNI_ENABLE_ORC=OFF \ + -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N + + # Arrow Java libraries + mvn install -Parrow-jni -P arrow-c-data -pl c,dataset -am \ + -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.cpp.build.dir=$ARROW_INSTALL_DIR/lib \ + -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly + popd +} diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh new file mode 100644 index 000000000000..221f01bf6fb5 --- /dev/null +++ b/dev/build_helper_functions.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function get_cxx_flags { + local CPU_ARCH=$1 + + local OS + OS=$(uname) + local MACHINE + MACHINE=$(uname -m) + ADDITIONAL_FLAGS="" + + if [[ -z "$CPU_ARCH" ]] || [[ $CPU_ARCH == "unknown" ]]; then + if [ "$OS" = "Darwin" ]; then + + if [ "$MACHINE" = "x86_64" ]; then + local CPU_CAPABILITIES + CPU_CAPABILITIES=$(sysctl -a | grep machdep.cpu.features | awk '{print tolower($0)}') + + if [[ $CPU_CAPABILITIES =~ "avx" ]]; then + CPU_ARCH="avx" + else + CPU_ARCH="sse" + fi + + elif [[ $(sysctl -a | grep machdep.cpu.brand_string) =~ "Apple" ]]; then + # Apple silicon. + CPU_ARCH="arm64" + fi + + # On MacOs prevent the flood of translation visibility settings warnings. + ADDITIONAL_FLAGS="-fvisibility=hidden -fvisibility-inlines-hidden" + else [ "$OS" = "Linux" ]; + + local CPU_CAPABILITIES + CPU_CAPABILITIES=$(cat /proc/cpuinfo | grep flags | head -n 1| awk '{print tolower($0)}') + + if [[ "$CPU_CAPABILITIES" =~ "avx" ]]; then + CPU_ARCH="avx" + elif [[ "$CPU_CAPABILITIES" =~ "sse" ]]; then + CPU_ARCH="sse" + elif [ "$MACHINE" = "aarch64" ]; then + CPU_ARCH="aarch64" + fi + fi + fi + + case $CPU_ARCH in + + "arm64") + echo -n "-mcpu=apple-m1+crc -std=c++17 -fvisibility=hidden $ADDITIONAL_FLAGS" + ;; + + "avx") + echo -n "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17 -mbmi2 $ADDITIONAL_FLAGS" + ;; + + "sse") + echo -n "-msse4.2 -std=c++17 $ADDITIONAL_FLAGS" + ;; + + "aarch64") + echo -n "-mcpu=neoverse-n1 -std=c++17 $ADDITIONAL_FLAGS" + ;; + *) + echo -n "Architecture not supported!" + esac + +} + +function wget_and_untar { + local URL=$1 + local DIR=$2 + mkdir -p "${DIR}" + pushd "${DIR}" + curl -L "${URL}" > $2.tar.gz + tar -xz --strip-components=1 -f $2.tar.gz + popd +} + +function cmake_install { + local NAME=$(basename "$(pwd)") + local BINARY_DIR=_build + SUDO="${SUDO:-""}" + if [ -d "${BINARY_DIR}" ] && prompt "Do you want to rebuild ${NAME}?"; then + ${SUDO} rm -rf "${BINARY_DIR}" + fi + mkdir -p "${BINARY_DIR}" + CPU_TARGET="${CPU_TARGET:-unknown}" + COMPILER_FLAGS=$(get_cxx_flags $CPU_TARGET) + + # CMAKE_POSITION_INDEPENDENT_CODE is required so that Velox can be built into dynamic libraries \ + cmake -Wno-dev -B"${BINARY_DIR}" \ + -GNinja \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_CXX_STANDARD=17 \ + "${INSTALL_PREFIX+-DCMAKE_PREFIX_PATH=}${INSTALL_PREFIX-}" \ + "${INSTALL_PREFIX+-DCMAKE_INSTALL_PREFIX=}${INSTALL_PREFIX-}" \ + -DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" \ + -DBUILD_TESTING=OFF \ + "$@" + + cmake --build "${BINARY_DIR}" + ${SUDO} cmake --install "${BINARY_DIR}" +} diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 35976d37a036..d5e33e926d82 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -24,14 +24,13 @@ ENABLE_S3=OFF ENABLE_HDFS=OFF ENABLE_ABFS=OFF ENABLE_EP_CACHE=OFF -ARROW_ENABLE_CUSTOM_CODEC=OFF ENABLE_VCPKG=OFF RUN_SETUP_SCRIPT=ON VELOX_REPO="" VELOX_BRANCH="" VELOX_HOME="" VELOX_PARAMETER="" -COMPILE_ARROW_JAVA=ON +BUILD_ARROW=ON SPARK_VERSION=ALL # set default number of threads as cpu cores minus 2 @@ -72,12 +71,10 @@ do ;; --enable_qat=*) ENABLE_QAT=("${arg#*=}") - ARROW_ENABLE_CUSTOM_CODEC=("${arg#*=}") shift # Remove argument name from processing ;; --enable_iaa=*) ENABLE_IAA=("${arg#*=}") - ARROW_ENABLE_CUSTOM_CODEC=("${arg#*=}") shift # Remove argument name from processing ;; --enable_hbm=*) @@ -136,8 +133,8 @@ do BUILD_VELOX_BENCHMARKS=("${arg#*=}") shift # Remove argument name from processing ;; - --compile_arrow_java=*) - COMPILE_ARROW_JAVA=("${arg#*=}") + --build_arrow=*) + BUILD_ARROW=("${arg#*=}") shift # Remove argument name from processing ;; --num_threads=*) @@ -189,6 +186,18 @@ fi concat_velox_param +function build_arrow { + echo "Start to build Arrow" + export SUDO=sudo + cd $GLUTEN_DIR/dev + source build_arrow.sh + prepare_arrow_build + build_arrow_cpp $BUILD_TYPE + echo "Finished building arrow CPP" + build_arrow_java + echo "Finished building arrow Java" +} + function build_velox { echo "Start to build Velox" cd $GLUTEN_DIR/ep/build-velox/src @@ -196,7 +205,7 @@ function build_velox { # When BUILD_TESTS is on for gluten cpp, we need turn on VELOX_BUILD_TEST_UTILS via build_test_utils. ./build_velox.sh --run_setup_script=$RUN_SETUP_SCRIPT --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ --enable_abfs=$ENABLE_ABFS --enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS \ - --compile_arrow_java=$COMPILE_ARROW_JAVA --num_threads=$NUM_THREADS + --num_threads=$NUM_THREADS } ## compile gluten cpp @@ -208,11 +217,15 @@ function build_gluten_cpp { cd build cmake -DBUILD_VELOX_BACKEND=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DBUILD_TESTS=$BUILD_TESTS -DBUILD_EXAMPLES=$BUILD_EXAMPLES -DBUILD_BENCHMARKS=$BUILD_BENCHMARKS -DBUILD_JEMALLOC=$BUILD_JEMALLOC \ - -DENABLE_HBM=$ENABLE_HBM -DENABLE_QAT=$ENABLE_QAT -DENABLE_IAA=$ENABLE_IAA -DENABLE_GCS=$ENABLE_GCS -DENABLE_S3=$ENABLE_S3 -DENABLE_HDFS=$ENABLE_HDFS -DENABLE_ABFS=$ENABLE_ABFS .. + -DENABLE_HBM=$ENABLE_HBM -DENABLE_QAT=$ENABLE_QAT -DENABLE_IAA=$ENABLE_IAA -DBUILD_PROTOBUF=$BUILD_PROTOBUF -DENABLE_GCS=$ENABLE_GCS \ + -DENABLE_S3=$ENABLE_S3 -DENABLE_HDFS=$ENABLE_HDFS -DENABLE_ABFS=$ENABLE_ABFS .. make -j $NUM_THREADS } function build_velox_backend { + if [ $BUILD_ARROW == "ON" ]; then + build_arrow + fi build_velox build_gluten_cpp } diff --git a/ep/build-velox/src/build_arrow_deps_centos8.sh b/ep/build-velox/src/build_arrow_deps_centos8.sh deleted file mode 100755 index 8dfc2af9375e..000000000000 --- a/ep/build-velox/src/build_arrow_deps_centos8.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -exu - -NPROC=$(getconf _NPROCESSORS_ONLN) - -function wget_and_untar { - local URL=$1 - local DIR=$2 - mkdir -p "${DIR}" - pushd "${DIR}" - curl -L "${URL}" > $2.tar.gz - tar -xz --strip-components=1 -f $2.tar.gz - popd -} - -function install_openssl { - wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl - cd openssl - ./config no-shared && make depend && make && sudo make install - cd .. -} - -function install_arrow_deps { - install_openssl -} - -# Activate gcc9; enable errors on unset variables afterwards. -# source /opt/rh/gcc-toolset-9/enable || exit 1 -install_arrow_deps -echo "All dependencies for Arrow installed!" diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 0224e9546861..97b3db5549bc 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -34,7 +34,6 @@ ENABLE_TESTS=OFF # Set to ON for gluten cpp test build. BUILD_TEST_UTILS=OFF RUN_SETUP_SCRIPT=ON -COMPILE_ARROW_JAVA=ON NUM_THREADS="" OTHER_ARGUMENTS="" @@ -87,10 +86,6 @@ for arg in "$@"; do RUN_SETUP_SCRIPT=("${arg#*=}") shift # Remove argument name from processing ;; - --compile_arrow_java=*) - COMPILE_ARROW_JAVA=("${arg#*=}") - shift # Remove argument name from processing - ;; --num_threads=*) NUM_THREADS=("${arg#*=}") shift # Remove argument name from processing @@ -191,7 +186,7 @@ function get_build_summary { echo "ENABLE_S3=$ENABLE_S3,ENABLE_GCS=$ENABLE_GCS,ENABLE_HDFS=$ENABLE_HDFS,ENABLE_ABFS=$ENABLE_ABFS,\ BUILD_TYPE=$BUILD_TYPE,VELOX_HOME=$VELOX_HOME,ENABLE_BENCHMARK=$ENABLE_BENCHMARK,\ ENABLE_TESTS=$ENABLE_TESTS,BUILD_TEST_UTILS=$BUILD_TEST_UTILS,\ -COMPILE_ARROW_JAVA=$COMPILE_ARROW_JAVA,OTHER_ARGUMENTS=$OTHER_ARGUMENTS,COMMIT_HASH=$COMMIT_HASH" +OTHER_ARGUMENTS=$OTHER_ARGUMENTS,COMMIT_HASH=$COMMIT_HASH" } function check_commit { @@ -277,34 +272,6 @@ function setup_linux { fi } -function compile_arrow_java_module() { - ARROW_HOME="${VELOX_HOME}/_build/$COMPILE_TYPE/third_party/arrow_ep/src/arrow_ep" - ARROW_INSTALL_DIR="${ARROW_HOME}/../../install" - - pushd $ARROW_HOME/java - # Because arrow-bom module need the -DprocessAllModules - mvn versions:set -DnewVersion=15.0.0-gluten -DprocessAllModules - - mvn clean install -pl bom,maven/module-info-compiler-maven-plugin,vector -am \ - -DskipTests -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly - - # Arrow C Data Interface CPP libraries - mvn generate-resources -P generate-libs-cdata-all-os -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR \ - -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N - - # Arrow JNI Date Interface CPP libraries - export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} - mvn generate-resources -Pgenerate-libs-jni-macos-linux -N -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR \ - -DARROW_GANDIVA=OFF -DARROW_JAVA_JNI_ENABLE_GANDIVA=OFF -DARROW_ORC=OFF -DARROW_JAVA_JNI_ENABLE_ORC=OFF \ - -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N - - # Arrow Java libraries - mvn install -Parrow-jni -P arrow-c-data -pl c,dataset -am \ - -Darrow.c.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR/lib -Darrow.cpp.build.dir=$ARROW_INSTALL_DIR/lib \ - -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -Dassembly.skipAssembly - popd -} - CURRENT_DIR=$( cd "$(dirname "$BASH_SOURCE")" pwd @@ -334,9 +301,5 @@ echo "Target Velox build: $TARGET_BUILD_SUMMARY" check_commit compile -if [ $COMPILE_ARROW_JAVA == "ON" ]; then - compile_arrow_java_module -fi - echo "Successfully built Velox from Source." echo $TARGET_BUILD_SUMMARY >"${VELOX_HOME}/velox-build.cache" From 369d01a5c44a074ced345013815f4aa25cf66aac Mon Sep 17 00:00:00 2001 From: exmy Date: Fri, 28 Jun 2024 14:52:31 +0800 Subject: [PATCH 360/402] [CORE] Remap the name of LOG/LOGARITHM (#6266) --- cpp-ch/local-engine/Parser/scalar_function_parser/ln.cpp | 2 +- cpp-ch/local-engine/Parser/scalar_function_parser/log.cpp | 2 +- .../scala/org/apache/gluten/expression/ExpressionNames.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/ln.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/ln.cpp index 081444ee5f4a..0bb19dd1d206 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/ln.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/ln.cpp @@ -25,7 +25,7 @@ class FunctionParserLn : public FunctionParserLogBase explicit FunctionParserLn(SerializedPlanParser * plan_parser_) : FunctionParserLogBase(plan_parser_) {} ~FunctionParserLn() override = default; - static constexpr auto name = "log"; + static constexpr auto name = "ln"; String getName() const override { return name; } String getCHFunctionName() const override { return name; } diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/log.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/log.cpp index 264c0da930c6..bafca3b213d7 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/log.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/log.cpp @@ -38,7 +38,7 @@ class FunctionParserLog : public FunctionParser explicit FunctionParserLog(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} ~FunctionParserLog() override = default; - static constexpr auto name = "logarithm"; + static constexpr auto name = "log"; String getName() const override { return name; } diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 8317e28b58bb..87b1b4e7539b 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -174,8 +174,8 @@ object ExpressionNames { final val SIGN = "sign" final val LOG1P = "log1p" final val LOG2 = "log2" - final val LOG = "log" - final val LOGARITHM = "logarithm" + final val LOG = "ln" + final val LOGARITHM = "log" final val RADIANS = "radians" final val GREATEST = "greatest" final val LEAST = "least" From e684bf3563e3d5cb4bed2217592c0a406c47d2ae Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Fri, 28 Jun 2024 16:07:48 +0800 Subject: [PATCH 361/402] [VL] Daily Update Velox Version (2024_06_28) (#6261) cc6ac918c by yanngyoung, Memory Arbitration Fuzzer test enable global arbitration (#10333) a34e7cd3f by zhli1142015, Support complex types for Spark EqualTo and EqualNullSafe functions (#10156) bb3520277 by Kevin Wilfong, Don't run plans with LocalPartition in Aggregation and Window Fuzzers when TSAN is enabled (#10340) a9fa01ff3 by Daniel Hunte, Add map_top_n_keys Presto function (#10271) 81b848499 by Kevin Wilfong, Speed up ApproxPercentileResultVerifier >100x (#10329) cf655ff01 by Andrii Rosa, Reduce number of explicit OutputBuffer#acknowledge calls (#9468) a83839960 by Andrii Rosa, Increase exchange client long pool delay to 10 seconds (#10287) 91f3aec1c by Masha Basmanova, Add partial support for split_to_map Presto lambda function (#10334) c3f2158a9 by Masha Basmanova, Add support for BOOLEAN input to approx_most_frequent Presto aggregate function (#10337) 9c56ef3d3 by Jimmy Lu, Enable non-flat vector for approx_percentile percentile array elements (#10324) d26cb1df9 by Zac Wen, Add fuzzer for async data cache (#10244) 5db5747c4 by Masha Basmanova, Add support for VARBINARY inputs to substr Presto function (#10332) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 237757d818d0..0adc1ce8ff61 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_27 +VELOX_BRANCH=2024_06_28 VELOX_HOME="" #Set on run gluten on HDFS From 86449d0c87437002d6d2636e079420cacb258d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Fri, 28 Jun 2024 16:22:37 +0800 Subject: [PATCH 362/402] [CORE][VL] Add OffloadProject to offload project having input_file_name's support considered (#6200) --- .../backendsapi/velox/VeloxBackend.scala | 2 + .../backendsapi/velox/VeloxIteratorApi.scala | 8 - .../velox/VeloxSparkPlanExecApi.scala | 7 +- .../extension/InputFileNameReplaceRule.scala | 155 ------------------ .../ScalarFunctionsValidateSuite.scala | 10 +- .../backendsapi/BackendSettingsApi.scala | 1 + .../columnar/MiscColumnarRules.scala | 2 +- .../columnar/OffloadSingleNode.scala | 141 +++++++++++++++- .../enumerated/EnumeratedTransform.scala | 1 + .../enumerated/RasOffloadProject.scala | 33 ++++ .../sql/GlutenColumnExpressionSuite.scala | 38 ++++- .../sql/GlutenColumnExpressionSuite.scala | 38 ++++- .../sql/GlutenColumnExpressionSuite.scala | 38 ++++- .../sql/GlutenColumnExpressionSuite.scala | 38 ++++- .../org/apache/gluten/GlutenConfig.scala | 11 -- .../sql/shims/spark32/Spark32Shims.scala | 11 +- .../sql/shims/spark33/Spark33Shims.scala | 3 + .../sql/shims/spark34/Spark34Shims.scala | 5 +- .../sql/shims/spark35/Spark35Shims.scala | 5 +- 19 files changed, 343 insertions(+), 204 deletions(-) delete mode 100644 backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 158be10f486c..82b45f2d4394 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -303,6 +303,8 @@ object VeloxBackendSettings extends BackendSettingsApi { override def supportNativeRowIndexColumn(): Boolean = true + override def supportNativeInputFileRelatedExpr(): Boolean = true + override def supportExpandExec(): Boolean = true override def supportSortExec(): Boolean = true diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 22862156c6b2..613e539456ec 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -19,7 +19,6 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.GlutenNumaBindingInfo import org.apache.gluten.backendsapi.IteratorApi import org.apache.gluten.execution._ -import org.apache.gluten.extension.InputFileNameReplaceRule import org.apache.gluten.metrics.IMetrics import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.substrait.plan.PlanNode @@ -134,13 +133,6 @@ class VeloxIteratorApi extends IteratorApi with Logging { } val metadataColumn = SparkShimLoader.getSparkShims.generateMetadataColumns(file, metadataColumnNames) - metadataColumn.put(InputFileNameReplaceRule.replacedInputFileName, file.filePath.toString) - metadataColumn.put( - InputFileNameReplaceRule.replacedInputFileBlockStart, - file.start.toString) - metadataColumn.put( - InputFileNameReplaceRule.replacedInputFileBlockLength, - file.length.toString) metadataColumns.add(metadataColumn) val partitionColumn = new JHashMap[String, String]() for (i <- 0 until file.partitionValues.numFields) { diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index b48da15683e8..ed69a5893c25 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -806,12 +806,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { * @return */ override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = { - val buf: ListBuffer[SparkSession => Rule[SparkPlan]] = - ListBuffer(BloomFilterMightContainJointRewriteRule.apply, ArrowScanReplaceRule.apply) - if (GlutenConfig.getConf.enableInputFileNameReplaceRule) { - buf += InputFileNameReplaceRule.apply - } - buf.result + List(BloomFilterMightContainJointRewriteRule.apply, ArrowScanReplaceRule.apply) } /** diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala deleted file mode 100644 index cd3f50d8e77f..000000000000 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/InputFileNameReplaceRule.scala +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, NamedExpression} -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.execution.{FileSourceScanExec, ProjectExec, SparkPlan} -import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan -import org.apache.spark.sql.types.{LongType, StringType} - -object InputFileNameReplaceRule { - val replacedInputFileName = "$input_file_name$" - val replacedInputFileBlockStart = "$input_file_block_start$" - val replacedInputFileBlockLength = "$input_file_block_length$" -} - -case class InputFileNameReplaceRule(spark: SparkSession) extends Rule[SparkPlan] { - import InputFileNameReplaceRule._ - - private def isInputFileName(expr: Expression): Boolean = { - expr match { - case _: InputFileName => true - case _ => false - } - } - - private def isInputFileBlockStart(expr: Expression): Boolean = { - expr match { - case _: InputFileBlockStart => true - case _ => false - } - } - - private def isInputFileBlockLength(expr: Expression): Boolean = { - expr match { - case _: InputFileBlockLength => true - case _ => false - } - } - - override def apply(plan: SparkPlan): SparkPlan = { - val replacedExprs = scala.collection.mutable.Map[String, AttributeReference]() - - def hasParquetScan(plan: SparkPlan): Boolean = { - plan match { - case fileScan: FileSourceScanExec - if fileScan.relation.fileFormat.isInstanceOf[ParquetFileFormat] => - true - case batchScan: BatchScanExec => - batchScan.scan match { - case _: ParquetScan => true - case _ => false - } - case _ => plan.children.exists(hasParquetScan) - } - } - - def mayNeedConvert(expr: Expression): Boolean = { - expr match { - case e if isInputFileName(e) => true - case s if isInputFileBlockStart(s) => true - case l if isInputFileBlockLength(l) => true - case other => other.children.exists(mayNeedConvert) - } - } - - def doConvert(expr: Expression): Expression = { - expr match { - case e if isInputFileName(e) => - replacedExprs.getOrElseUpdate( - replacedInputFileName, - AttributeReference(replacedInputFileName, StringType, true)()) - case s if isInputFileBlockStart(s) => - replacedExprs.getOrElseUpdate( - replacedInputFileBlockStart, - AttributeReference(replacedInputFileBlockStart, LongType, true)() - ) - case l if isInputFileBlockLength(l) => - replacedExprs.getOrElseUpdate( - replacedInputFileBlockLength, - AttributeReference(replacedInputFileBlockLength, LongType, true)() - ) - case other => - other.withNewChildren(other.children.map(child => doConvert(child))) - } - } - - def ensureChildOutputHasNewAttrs(plan: SparkPlan): SparkPlan = { - plan match { - case _ @ProjectExec(projectList, child) => - var newProjectList = projectList - for ((_, newAttr) <- replacedExprs) { - if (!newProjectList.exists(attr => attr.exprId == newAttr.exprId)) { - newProjectList = newProjectList :+ newAttr.toAttribute - } - } - val newChild = ensureChildOutputHasNewAttrs(child) - ProjectExec(newProjectList, newChild) - case f: FileSourceScanExec => - var newOutput = f.output - for ((_, newAttr) <- replacedExprs) { - if (!newOutput.exists(attr => attr.exprId == newAttr.exprId)) { - newOutput = newOutput :+ newAttr.toAttribute - } - } - f.copy(output = newOutput) - - case b: BatchScanExec => - var newOutput = b.output - for ((_, newAttr) <- replacedExprs) { - if (!newOutput.exists(attr => attr.exprId == newAttr.exprId)) { - newOutput = newOutput :+ newAttr - } - } - b.copy(output = newOutput) - case other => - val newChildren = other.children.map(ensureChildOutputHasNewAttrs) - other.withNewChildren(newChildren) - } - } - - def replaceInputFileNameInProject(plan: SparkPlan): SparkPlan = { - plan match { - case _ @ProjectExec(projectList, child) - if projectList.exists(mayNeedConvert) && hasParquetScan(plan) => - val newProjectList = projectList.map { - expr => doConvert(expr).asInstanceOf[NamedExpression] - } - val newChild = replaceInputFileNameInProject(ensureChildOutputHasNewAttrs(child)) - ProjectExec(newProjectList, newChild) - case other => - val newChildren = other.children.map(replaceInputFileNameInProject) - other.withNewChildren(newChildren) - } - } - replaceInputFileNameInProject(plan) - } -} diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index a2baf95ecdc0..bd32a799c3ac 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -645,13 +645,9 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } test("Test input_file_name function") { - withSQLConf( - "spark.gluten.sql.enableInputFileNameReplaceRule" -> "true" - ) { - runQueryAndCompare("""SELECT input_file_name(), l_orderkey - | from lineitem limit 100""".stripMargin) { - checkGlutenOperatorMatch[ProjectExecTransformer] - } + runQueryAndCompare("""SELECT input_file_name(), l_orderkey + | from lineitem limit 100""".stripMargin) { + checkGlutenOperatorMatch[ProjectExecTransformer] } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index b132366e6e1d..50292839b684 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -44,6 +44,7 @@ trait BackendSettingsApi { def supportNativeWrite(fields: Array[StructField]): Boolean = true def supportNativeMetadataColumns(): Boolean = false def supportNativeRowIndexColumn(): Boolean = false + def supportNativeInputFileRelatedExpr(): Boolean = false def supportExpandExec(): Boolean = false def supportSortExec(): Boolean = false diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala index 8ed2137f4489..15fc8bea7054 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala @@ -30,7 +30,7 @@ object MiscColumnarRules { object TransformPreOverrides { def apply(): TransformPreOverrides = { TransformPreOverrides( - List(OffloadFilter()), + List(OffloadProject(), OffloadFilter()), List( OffloadOthers(), OffloadAggregate(), diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 75da28e30d39..11db0bc1faf1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -26,15 +26,19 @@ import org.apache.gluten.utils.{LogLevelUtil, PlanUtil} import org.apache.spark.api.python.EvalPythonExecTransformer import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, NamedExpression} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} import org.apache.spark.sql.execution.datasources.WriteFilesExec -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2ScanExecBase} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins._ import org.apache.spark.sql.execution.python.{ArrowEvalPythonExec, BatchEvalPythonExec} import org.apache.spark.sql.execution.window.{WindowExec, WindowGroupLimitExecShim} import org.apache.spark.sql.hive.HiveTableScanExecTransformer +import org.apache.spark.sql.types.{LongType, StringType} + +import scala.collection.mutable.Map /** * Converts a vanilla Spark plan node into Gluten plan node. Gluten plan is supposed to be executed @@ -181,7 +185,138 @@ case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { case other => other } } +} + +case class OffloadProject() extends OffloadSingleNode with LogLevelUtil { + private def containsInputFileRelatedExpr(expr: Expression): Boolean = { + expr match { + case _: InputFileName | _: InputFileBlockStart | _: InputFileBlockLength => true + case _ => expr.children.exists(containsInputFileRelatedExpr) + } + } + + private def rewriteExpr( + expr: Expression, + replacedExprs: Map[String, AttributeReference]): Expression = { + expr match { + case _: InputFileName => + replacedExprs.getOrElseUpdate( + expr.prettyName, + AttributeReference(expr.prettyName, StringType, false)()) + case _: InputFileBlockStart => + replacedExprs.getOrElseUpdate( + expr.prettyName, + AttributeReference(expr.prettyName, LongType, false)()) + case _: InputFileBlockLength => + replacedExprs.getOrElseUpdate( + expr.prettyName, + AttributeReference(expr.prettyName, LongType, false)()) + case other => + other.withNewChildren(other.children.map(child => rewriteExpr(child, replacedExprs))) + } + } + + private def addMetadataCol( + plan: SparkPlan, + replacedExprs: Map[String, AttributeReference]): SparkPlan = { + def genNewOutput(output: Seq[Attribute]): Seq[Attribute] = { + var newOutput = output + for ((_, newAttr) <- replacedExprs) { + if (!newOutput.exists(attr => attr.exprId == newAttr.exprId)) { + newOutput = newOutput :+ newAttr + } + } + newOutput + } + def genNewProjectList(projectList: Seq[NamedExpression]): Seq[NamedExpression] = { + var newProjectList = projectList + for ((_, newAttr) <- replacedExprs) { + if (!newProjectList.exists(attr => attr.exprId == newAttr.exprId)) { + newProjectList = newProjectList :+ newAttr.toAttribute + } + } + newProjectList + } + + plan match { + case f: FileSourceScanExec => + f.copy(output = genNewOutput(f.output)) + case f: FileSourceScanExecTransformer => + f.copy(output = genNewOutput(f.output)) + case b: BatchScanExec => + b.copy(output = genNewOutput(b.output).asInstanceOf[Seq[AttributeReference]]) + case b: BatchScanExecTransformer => + b.copy(output = genNewOutput(b.output).asInstanceOf[Seq[AttributeReference]]) + case p @ ProjectExec(projectList, child) => + p.copy(genNewProjectList(projectList), addMetadataCol(child, replacedExprs)) + case p @ ProjectExecTransformer(projectList, child) => + p.copy(genNewProjectList(projectList), addMetadataCol(child, replacedExprs)) + case _ => plan.withNewChildren(plan.children.map(addMetadataCol(_, replacedExprs))) + } + } + + private def tryOffloadProjectExecWithInputFileRelatedExprs( + projectExec: ProjectExec): SparkPlan = { + def findScanNodes(plan: SparkPlan): Seq[SparkPlan] = { + plan.collect { + case f @ (_: FileSourceScanExec | _: AbstractFileSourceScanExec | + _: DataSourceV2ScanExecBase) => + f + } + } + val addHint = AddTransformHintRule() + val newProjectList = projectExec.projectList.filterNot(containsInputFileRelatedExpr) + val newProjectExec = ProjectExec(newProjectList, projectExec.child) + addHint.apply(newProjectExec) + if (TransformHints.isNotTransformable(newProjectExec)) { + // Project is still not transformable after remove `input_file_name` expressions. + projectExec + } else { + // the project with `input_file_name` expression should have at most + // one data source, reference: + // https://github.com/apache/spark/blob/e459674127e7b21e2767cc62d10ea6f1f941936c + // /sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala#L506 + val leafScans = findScanNodes(projectExec) + assert(leafScans.size <= 1) + if (leafScans.isEmpty || TransformHints.isNotTransformable(leafScans(0))) { + // It means + // 1. projectExec has `input_file_name` but no scan child. + // 2. It has scan child node but the scan node fallback. + projectExec + } else { + val replacedExprs = scala.collection.mutable.Map[String, AttributeReference]() + val newProjectList = projectExec.projectList.map { + expr => rewriteExpr(expr, replacedExprs).asInstanceOf[NamedExpression] + } + val newChild = addMetadataCol(projectExec.child, replacedExprs) + logDebug( + s"Columnar Processing for ${projectExec.getClass} with " + + s"ProjectList ${projectExec.projectList} is currently supported.") + ProjectExecTransformer(newProjectList, newChild) + } + } + } + private def genProjectExec(projectExec: ProjectExec): SparkPlan = { + if ( + TransformHints.isNotTransformable(projectExec) && + BackendsApiManager.getSettings.supportNativeInputFileRelatedExpr() && + projectExec.projectList.exists(containsInputFileRelatedExpr) + ) { + tryOffloadProjectExecWithInputFileRelatedExprs(projectExec) + } else if (TransformHints.isNotTransformable(projectExec)) { + projectExec + } else { + logDebug(s"Columnar Processing for ${projectExec.getClass} is currently supported.") + ProjectExecTransformer(projectExec.projectList, projectExec.child) + } + } + + override def offload(plan: SparkPlan): SparkPlan = plan match { + case p: ProjectExec => + genProjectExec(p) + case other => other + } } // Filter transformation. @@ -261,10 +396,6 @@ object OffloadOthers { case plan: CoalesceExec => logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") ColumnarCoalesceExec(plan.numPartitions, plan.child) - case plan: ProjectExec => - val columnarChild = plan.child - logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - ProjectExecTransformer(plan.projectList, columnarChild) case plan: SortAggregateExec => logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") HashAggregateExecBaseTransformer.from(plan) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index 9a54a101453f..30e4c0a79823 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -54,6 +54,7 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) RasOffload.from[BaseJoinExec](OffloadJoin()).toRule, RasOffloadHashAggregate.toRule, RasOffloadFilter.toRule, + RasOffloadProject.toRule, RasOffload.from[DataSourceV2ScanExecBase](OffloadOthers()).toRule, RasOffload.from[DataSourceScanExec](OffloadOthers()).toRule, RasOffload diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala new file mode 100644 index 000000000000..0bbf57499b73 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffloadProject.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.enumerated + +import org.apache.gluten.execution.ProjectExecTransformer + +import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} + +object RasOffloadProject extends RasOffload { + override def offload(node: SparkPlan): SparkPlan = node match { + case ProjectExec(projectList, child) => + ProjectExecTransformer(projectList, child) + case other => + other + } + + override def typeIdentifier(): RasOffload.TypeIdentifier = + RasOffload.TypeIdentifier.of[ProjectExec] +} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala index edd2a5a9672d..a4b530e637af 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala @@ -16,4 +16,40 @@ */ package org.apache.spark.sql -class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.execution.ProjectExec +import org.apache.spark.sql.functions.{expr, input_file_name} +import org.apache.spark.sql.types.{ArrayType, IntegerType, StringType, StructField, StructType} + +class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait { + testGluten("input_file_name with scan is fallback") { + withTempPath { + dir => + val rawData = Seq( + Row(1, "Alice", Seq(Row(Seq(1, 2, 3)))), + Row(2, "Bob", Seq(Row(Seq(4, 5)))), + Row(3, "Charlie", Seq(Row(Seq(6, 7, 8, 9)))) + ) + val schema = StructType( + Array( + StructField("id", IntegerType, nullable = false), + StructField("name", StringType, nullable = false), + StructField( + "nested_column", + ArrayType( + StructType(Array( + StructField("array_in_struct", ArrayType(IntegerType), nullable = true) + ))), + nullable = true) + )) + val data: DataFrame = spark.createDataFrame(sparkContext.parallelize(rawData), schema) + data.write.parquet(dir.getCanonicalPath) + + val q = + spark.read.parquet(dir.getCanonicalPath).select(input_file_name(), expr("nested_column")) + val firstRow = q.head() + assert(firstRow.getString(0).contains(dir.toURI.getPath)) + val project = q.queryExecution.executedPlan.collect { case p: ProjectExec => p } + assert(project.size == 1) + } + } +} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala index edd2a5a9672d..a4b530e637af 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala @@ -16,4 +16,40 @@ */ package org.apache.spark.sql -class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.execution.ProjectExec +import org.apache.spark.sql.functions.{expr, input_file_name} +import org.apache.spark.sql.types.{ArrayType, IntegerType, StringType, StructField, StructType} + +class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait { + testGluten("input_file_name with scan is fallback") { + withTempPath { + dir => + val rawData = Seq( + Row(1, "Alice", Seq(Row(Seq(1, 2, 3)))), + Row(2, "Bob", Seq(Row(Seq(4, 5)))), + Row(3, "Charlie", Seq(Row(Seq(6, 7, 8, 9)))) + ) + val schema = StructType( + Array( + StructField("id", IntegerType, nullable = false), + StructField("name", StringType, nullable = false), + StructField( + "nested_column", + ArrayType( + StructType(Array( + StructField("array_in_struct", ArrayType(IntegerType), nullable = true) + ))), + nullable = true) + )) + val data: DataFrame = spark.createDataFrame(sparkContext.parallelize(rawData), schema) + data.write.parquet(dir.getCanonicalPath) + + val q = + spark.read.parquet(dir.getCanonicalPath).select(input_file_name(), expr("nested_column")) + val firstRow = q.head() + assert(firstRow.getString(0).contains(dir.toURI.getPath)) + val project = q.queryExecution.executedPlan.collect { case p: ProjectExec => p } + assert(project.size == 1) + } + } +} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala index edd2a5a9672d..a4b530e637af 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala @@ -16,4 +16,40 @@ */ package org.apache.spark.sql -class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.execution.ProjectExec +import org.apache.spark.sql.functions.{expr, input_file_name} +import org.apache.spark.sql.types.{ArrayType, IntegerType, StringType, StructField, StructType} + +class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait { + testGluten("input_file_name with scan is fallback") { + withTempPath { + dir => + val rawData = Seq( + Row(1, "Alice", Seq(Row(Seq(1, 2, 3)))), + Row(2, "Bob", Seq(Row(Seq(4, 5)))), + Row(3, "Charlie", Seq(Row(Seq(6, 7, 8, 9)))) + ) + val schema = StructType( + Array( + StructField("id", IntegerType, nullable = false), + StructField("name", StringType, nullable = false), + StructField( + "nested_column", + ArrayType( + StructType(Array( + StructField("array_in_struct", ArrayType(IntegerType), nullable = true) + ))), + nullable = true) + )) + val data: DataFrame = spark.createDataFrame(sparkContext.parallelize(rawData), schema) + data.write.parquet(dir.getCanonicalPath) + + val q = + spark.read.parquet(dir.getCanonicalPath).select(input_file_name(), expr("nested_column")) + val firstRow = q.head() + assert(firstRow.getString(0).contains(dir.toURI.getPath)) + val project = q.queryExecution.executedPlan.collect { case p: ProjectExec => p } + assert(project.size == 1) + } + } +} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala index edd2a5a9672d..8a28c4e98a26 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala @@ -16,4 +16,40 @@ */ package org.apache.spark.sql -class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait {} +import org.apache.spark.sql.execution.ProjectExec +import org.apache.spark.sql.functions.{expr, input_file_name} +import org.apache.spark.sql.types._ + +class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait { + testGluten("input_file_name with scan is fallback") { + withTempPath { + dir => + val rawData = Seq( + Row(1, "Alice", Seq(Row(Seq(1, 2, 3)))), + Row(2, "Bob", Seq(Row(Seq(4, 5)))), + Row(3, "Charlie", Seq(Row(Seq(6, 7, 8, 9)))) + ) + val schema = StructType( + Array( + StructField("id", IntegerType, nullable = false), + StructField("name", StringType, nullable = false), + StructField( + "nested_column", + ArrayType( + StructType(Array( + StructField("array_in_struct", ArrayType(IntegerType), nullable = true) + ))), + nullable = true) + )) + val data: DataFrame = spark.createDataFrame(sparkContext.parallelize(rawData), schema) + data.write.parquet(dir.getCanonicalPath) + + val q = + spark.read.parquet(dir.getCanonicalPath).select(input_file_name(), expr("nested_column")) + val firstRow = q.head() + assert(firstRow.getString(0).contains(dir.toURI.getPath)) + val project = q.queryExecution.executedPlan.collect { case p: ProjectExec => p } + assert(project.size == 1) + } + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 58b99a7f3064..ec80ba86a7b9 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -37,7 +37,6 @@ case class GlutenNumaBindingInfo( class GlutenConfig(conf: SQLConf) extends Logging { import GlutenConfig._ - def enableInputFileNameReplaceRule: Boolean = conf.getConf(INPUT_FILE_NAME_REPLACE_RULE_ENABLED) def enableAnsiMode: Boolean = conf.ansiEnabled def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED) @@ -767,16 +766,6 @@ object GlutenConfig { .booleanConf .createWithDefault(GLUTEN_ENABLE_BY_DEFAULT) - val INPUT_FILE_NAME_REPLACE_RULE_ENABLED = - buildConf("spark.gluten.sql.enableInputFileNameReplaceRule") - .internal() - .doc( - "Experimental: This config apply for velox backend to specify whether to enable " + - "inputFileNameReplaceRule to support offload input_file_name " + - "expression to native.") - .booleanConf - .createWithDefault(false) - // FIXME the option currently controls both JVM and native validation against a Substrait plan. val NATIVE_VALIDATION_ENABLED = buildConf("spark.gluten.sql.enable.native.validation") diff --git a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala index b9c37ef3d730..b036d6dd9a41 100644 --- a/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala +++ b/shims/spark32/src/main/scala/org/apache/gluten/sql/shims/spark32/Spark32Shims.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.csv.CSVOptions -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, BinaryExpression, Expression} +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, BinaryExpression, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName} import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.physical.{Distribution, HashClusteredDistribution} @@ -236,8 +236,13 @@ class Spark32Shims extends SparkShims { override def generateMetadataColumns( file: PartitionedFile, - metadataColumnNames: Seq[String]): JMap[String, String] = - new JHashMap[String, String]() + metadataColumnNames: Seq[String]): JMap[String, String] = { + val metadataColumn = new JHashMap[String, String]() + metadataColumn.put(InputFileName().prettyName, file.filePath) + metadataColumn.put(InputFileBlockStart().prettyName, file.start.toString) + metadataColumn.put(InputFileBlockLength().prettyName, file.length.toString) + metadataColumn + } def getAnalysisExceptionPlan(ae: AnalysisException): Option[LogicalPlan] = { ae.plan diff --git a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala index d6292b46c261..8b12c2642c55 100644 --- a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala +++ b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala @@ -231,6 +231,9 @@ class Spark33Shims extends SparkShims { case _ => } } + metadataColumn.put(InputFileName().prettyName, file.filePath) + metadataColumn.put(InputFileBlockStart().prettyName, file.start.toString) + metadataColumn.put(InputFileBlockLength().prettyName, file.length.toString) metadataColumn } diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index c718f4ed25d6..420be8511937 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -240,8 +240,9 @@ class Spark34Shims extends SparkShims { case _ => } } - - // TODO: row_index metadata support + metadataColumn.put(InputFileName().prettyName, file.filePath.toString) + metadataColumn.put(InputFileBlockStart().prettyName, file.start.toString) + metadataColumn.put(InputFileBlockLength().prettyName, file.length.toString) metadataColumn } diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index f6feae01a8b2..8ac8d323efd6 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -238,8 +238,9 @@ class Spark35Shims extends SparkShims { case _ => } } - - // TODO row_index metadata support + metadataColumn.put(InputFileName().prettyName, file.filePath.toString) + metadataColumn.put(InputFileBlockStart().prettyName, file.start.toString) + metadataColumn.put(InputFileBlockLength().prettyName, file.length.toString) metadataColumn } From 0b34e8e941adc7eff84bba9eaaae6ea0e432ac77 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Mon, 1 Jul 2024 07:13:04 +0800 Subject: [PATCH 363/402] [GLUTEN-6253] Use internal udf config to avoid modify the original one (#6255) The current implementation sets spark.gluten.sql.columnar.backend.velox.udfLibraryPaths on driver side after resolving the library paths. This approach can overwrite the original settings with a local file path on the driver node before sending the SparkConf to all executors, and the executors on different nodes will fail while accessing that path. This PR sets the resolved library paths to an internal config to avoid the conflicts. Manually verified on a multi-node cluster. --- .../org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 1 + .../scala/org/apache/spark/sql/expression/UDFResolver.scala | 3 ++- cpp/velox/config/VeloxConfig.h | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 82b45f2d4394..0238508d9699 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -65,6 +65,7 @@ object VeloxBackendSettings extends BackendSettingsApi { val GLUTEN_VELOX_UDF_LIB_PATHS = getBackendConfigPrefix() + ".udfLibraryPaths" val GLUTEN_VELOX_DRIVER_UDF_LIB_PATHS = getBackendConfigPrefix() + ".driver.udfLibraryPaths" + val GLUTEN_VELOX_INTERNAL_UDF_LIB_PATHS = getBackendConfigPrefix() + ".internal.udfLibraryPaths" val MAXIMUM_BATCH_SIZE: Int = 32768 diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index 8a549c9b4ea9..99f9faf9914a 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -231,8 +231,9 @@ object UDFResolver extends Logging { udfLibPaths match { case Some(paths) => + // Set resolved paths to the internal config to parse on native side. sparkConf.set( - VeloxBackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, + VeloxBackendSettings.GLUTEN_VELOX_INTERNAL_UDF_LIB_PATHS, getAllLibraries(sparkConf, isDriver, paths)) case None => } diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h index f57f1293e22e..7a96f03f4985 100644 --- a/cpp/velox/config/VeloxConfig.h +++ b/cpp/velox/config/VeloxConfig.h @@ -99,7 +99,7 @@ const std::string kVeloxAsyncTimeoutOnTaskStopping = const int32_t kVeloxAsyncTimeoutOnTaskStoppingDefault = 30000; // 30s // udf -const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.udfLibraryPaths"; +const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.internal.udfLibraryPaths"; // backtrace allocation const std::string kBacktraceAllocation = "spark.gluten.backtrace.allocation"; From d516f5619410c3056da95f0f4fa7e85727502f16 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Mon, 1 Jul 2024 09:10:41 +0800 Subject: [PATCH 364/402] [CORE] Creates vanilla plan when the join operators fall back (#6093) --- .../backendsapi/clickhouse/CHBackend.scala | 1 - ...ouseTPCHNullableColumnarShuffleSuite.scala | 8 +- .../GlutenClickHouseTPCHNullableSuite.scala | 8 +- .../execution/GlutenClickHouseTPCHSuite.scala | 8 +- .../v1-bhj-ras/spark32/20.txt | 152 +++++---- .../v1-bhj-ras/spark33/20.txt | 152 +++++---- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 152 +++++---- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 152 +++++---- .../tpch-approved-plan/v1-ras/spark32/10.txt | 132 +++++--- .../tpch-approved-plan/v1-ras/spark32/11.txt | 102 +++--- .../tpch-approved-plan/v1-ras/spark32/12.txt | 64 ++-- .../tpch-approved-plan/v1-ras/spark32/13.txt | 70 ++-- .../tpch-approved-plan/v1-ras/spark32/14.txt | 50 +-- .../tpch-approved-plan/v1-ras/spark32/15.txt | 64 ++-- .../tpch-approved-plan/v1-ras/spark32/16.txt | 82 +++-- .../tpch-approved-plan/v1-ras/spark32/17.txt | 89 ++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 161 +++++---- .../tpch-approved-plan/v1-ras/spark32/19.txt | 50 +-- .../tpch-approved-plan/v1-ras/spark32/20.txt | 204 +++++++----- .../tpch-approved-plan/v1-ras/spark32/21.txt | 187 +++++++---- .../tpch-approved-plan/v1-ras/spark32/22.txt | 56 ++-- .../tpch-approved-plan/v1-ras/spark32/3.txt | 92 ++++-- .../tpch-approved-plan/v1-ras/spark32/4.txt | 66 ++-- .../tpch-approved-plan/v1-ras/spark32/5.txt | 204 +++++++----- .../tpch-approved-plan/v1-ras/spark32/7.txt | 196 +++++++---- .../tpch-approved-plan/v1-ras/spark32/8.txt | 270 +++++++++------ .../tpch-approved-plan/v1-ras/spark32/9.txt | 198 ++++++----- .../tpch-approved-plan/v1-ras/spark33/10.txt | 132 +++++--- .../tpch-approved-plan/v1-ras/spark33/11.txt | 312 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark33/12.txt | 64 ++-- .../tpch-approved-plan/v1-ras/spark33/13.txt | 70 ++-- .../tpch-approved-plan/v1-ras/spark33/14.txt | 50 +-- .../tpch-approved-plan/v1-ras/spark33/15.txt | 162 ++++----- .../tpch-approved-plan/v1-ras/spark33/16.txt | 82 +++-- .../tpch-approved-plan/v1-ras/spark33/17.txt | 89 ++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 161 +++++---- .../tpch-approved-plan/v1-ras/spark33/19.txt | 50 +-- .../tpch-approved-plan/v1-ras/spark33/20.txt | 204 +++++++----- .../tpch-approved-plan/v1-ras/spark33/21.txt | 187 +++++++---- .../tpch-approved-plan/v1-ras/spark33/22.txt | 130 ++++---- .../tpch-approved-plan/v1-ras/spark33/3.txt | 92 ++++-- .../tpch-approved-plan/v1-ras/spark33/4.txt | 66 ++-- .../tpch-approved-plan/v1-ras/spark33/5.txt | 204 +++++++----- .../tpch-approved-plan/v1-ras/spark33/7.txt | 196 +++++++---- .../tpch-approved-plan/v1-ras/spark33/8.txt | 270 +++++++++------ .../tpch-approved-plan/v1-ras/spark33/9.txt | 198 ++++++----- .../tpch-approved-plan/v1-ras/spark34/10.txt | 132 +++++--- .../tpch-approved-plan/v1-ras/spark34/11.txt | 312 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark34/12.txt | 64 ++-- .../tpch-approved-plan/v1-ras/spark34/13.txt | 70 ++-- .../tpch-approved-plan/v1-ras/spark34/14.txt | 50 +-- .../tpch-approved-plan/v1-ras/spark34/15.txt | 162 ++++----- .../tpch-approved-plan/v1-ras/spark34/16.txt | 82 +++-- .../tpch-approved-plan/v1-ras/spark34/17.txt | 89 ++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 161 +++++---- .../tpch-approved-plan/v1-ras/spark34/19.txt | 50 +-- .../tpch-approved-plan/v1-ras/spark34/20.txt | 204 +++++++----- .../tpch-approved-plan/v1-ras/spark34/21.txt | 187 +++++++---- .../tpch-approved-plan/v1-ras/spark34/22.txt | 130 ++++---- .../tpch-approved-plan/v1-ras/spark34/3.txt | 92 ++++-- .../tpch-approved-plan/v1-ras/spark34/4.txt | 66 ++-- .../tpch-approved-plan/v1-ras/spark34/5.txt | 204 +++++++----- .../tpch-approved-plan/v1-ras/spark34/7.txt | 196 +++++++---- .../tpch-approved-plan/v1-ras/spark34/8.txt | 270 +++++++++------ .../tpch-approved-plan/v1-ras/spark34/9.txt | 198 ++++++----- .../tpch-approved-plan/v1/spark32/10.txt | 132 +++++--- .../tpch-approved-plan/v1/spark32/11.txt | 102 +++--- .../tpch-approved-plan/v1/spark32/12.txt | 64 ++-- .../tpch-approved-plan/v1/spark32/13.txt | 70 ++-- .../tpch-approved-plan/v1/spark32/14.txt | 50 +-- .../tpch-approved-plan/v1/spark32/15.txt | 64 ++-- .../tpch-approved-plan/v1/spark32/16.txt | 82 +++-- .../tpch-approved-plan/v1/spark32/17.txt | 89 ++--- .../tpch-approved-plan/v1/spark32/18.txt | 161 +++++---- .../tpch-approved-plan/v1/spark32/19.txt | 50 +-- .../tpch-approved-plan/v1/spark32/20.txt | 204 +++++++----- .../tpch-approved-plan/v1/spark32/21.txt | 187 +++++++---- .../tpch-approved-plan/v1/spark32/22.txt | 56 ++-- .../tpch-approved-plan/v1/spark32/3.txt | 92 ++++-- .../tpch-approved-plan/v1/spark32/4.txt | 66 ++-- .../tpch-approved-plan/v1/spark32/5.txt | 204 +++++++----- .../tpch-approved-plan/v1/spark32/7.txt | 196 +++++++---- .../tpch-approved-plan/v1/spark32/8.txt | 270 +++++++++------ .../tpch-approved-plan/v1/spark32/9.txt | 198 ++++++----- .../tpch-approved-plan/v1/spark33/10.txt | 132 +++++--- .../tpch-approved-plan/v1/spark33/11.txt | 312 ++++++++++-------- .../tpch-approved-plan/v1/spark33/12.txt | 64 ++-- .../tpch-approved-plan/v1/spark33/13.txt | 70 ++-- .../tpch-approved-plan/v1/spark33/14.txt | 50 +-- .../tpch-approved-plan/v1/spark33/15.txt | 162 ++++----- .../tpch-approved-plan/v1/spark33/16.txt | 82 +++-- .../tpch-approved-plan/v1/spark33/17.txt | 89 ++--- .../tpch-approved-plan/v1/spark33/18.txt | 161 +++++---- .../tpch-approved-plan/v1/spark33/19.txt | 50 +-- .../tpch-approved-plan/v1/spark33/20.txt | 204 +++++++----- .../tpch-approved-plan/v1/spark33/21.txt | 187 +++++++---- .../tpch-approved-plan/v1/spark33/22.txt | 164 ++++----- .../tpch-approved-plan/v1/spark33/3.txt | 92 ++++-- .../tpch-approved-plan/v1/spark33/4.txt | 66 ++-- .../tpch-approved-plan/v1/spark33/5.txt | 204 +++++++----- .../tpch-approved-plan/v1/spark33/7.txt | 196 +++++++---- .../tpch-approved-plan/v1/spark33/8.txt | 270 +++++++++------ .../tpch-approved-plan/v1/spark33/9.txt | 198 ++++++----- .../tpch-approved-plan/v1/spark34/10.txt | 132 +++++--- .../tpch-approved-plan/v1/spark34/11.txt | 312 ++++++++++-------- .../tpch-approved-plan/v1/spark34/12.txt | 64 ++-- .../tpch-approved-plan/v1/spark34/13.txt | 70 ++-- .../tpch-approved-plan/v1/spark34/14.txt | 50 +-- .../tpch-approved-plan/v1/spark34/15.txt | 162 ++++----- .../tpch-approved-plan/v1/spark34/16.txt | 82 +++-- .../tpch-approved-plan/v1/spark34/17.txt | 89 ++--- .../tpch-approved-plan/v1/spark34/18.txt | 161 +++++---- .../tpch-approved-plan/v1/spark34/19.txt | 50 +-- .../tpch-approved-plan/v1/spark34/20.txt | 204 +++++++----- .../tpch-approved-plan/v1/spark34/21.txt | 187 +++++++---- .../tpch-approved-plan/v1/spark34/22.txt | 164 ++++----- .../tpch-approved-plan/v1/spark34/3.txt | 92 ++++-- .../tpch-approved-plan/v1/spark34/4.txt | 66 ++-- .../tpch-approved-plan/v1/spark34/5.txt | 204 +++++++----- .../tpch-approved-plan/v1/spark34/7.txt | 196 +++++++---- .../tpch-approved-plan/v1/spark34/8.txt | 270 +++++++++------ .../tpch-approved-plan/v1/spark34/9.txt | 198 ++++++----- .../gluten/execution/FallbackSuite.scala | 23 ++ .../org/apache/gluten/GlutenPlugin.scala | 3 +- .../backendsapi/BackendSettingsApi.scala | 1 - .../apache/gluten/execution/SortUtils.scala | 33 +- .../gluten/extension/StrategyOverrides.scala | 213 ------------ .../columnar/OffloadSingleNode.scala | 29 +- .../columnar/TransformHintRule.scala | 34 +- .../columnar/rewrite/RewriteJoin.scala | 63 ++++ .../columnar/rewrite/RewriteSingleNode.scala | 2 +- .../RewriteSparkPlanRulesManager.scala | 11 +- .../utils/velox/VeloxTestSettings.scala | 1 - .../joins/GlutenBroadcastJoinSuite.scala | 19 -- .../GlutenSessionExtensionSuite.scala | 3 +- .../utils/velox/VeloxTestSettings.scala | 1 - .../GlutenReplaceHashWithSortAggSuite.scala | 8 +- .../GlutenSessionExtensionSuite.scala | 3 +- .../utils/velox/VeloxTestSettings.scala | 4 - .../apache/spark/sql/GlutenJoinSuite.scala | 13 - .../GlutenReplaceHashWithSortAggSuite.scala | 8 +- .../GlutenSessionExtensionSuite.scala | 3 +- .../utils/velox/VeloxTestSettings.scala | 4 - .../apache/spark/sql/GlutenJoinSuite.scala | 13 - .../GlutenReplaceHashWithSortAggSuite.scala | 8 +- .../GlutenSessionExtensionSuite.scala | 3 +- .../org/apache/gluten/GlutenConfig.scala | 10 - .../sql/execution/JoinSelectionShim.scala | 51 --- .../sql/execution/JoinSelectionShim.scala | 51 --- .../sql/execution/JoinSelectionShim.scala | 51 --- 150 files changed, 9890 insertions(+), 7027 deletions(-) rename shims/spark32/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala => gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala (53%) delete mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala create mode 100644 gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala delete mode 100644 shims/spark33/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala delete mode 100644 shims/spark34/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala delete mode 100644 shims/spark35/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index 1587b9ea3488..cdca1b031a91 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -189,7 +189,6 @@ object CHBackendSettings extends BackendSettingsApi with Logging { } } - override def utilizeShuffledHashJoinHint(): Boolean = true override def supportShuffleWithProject( outputPartitioning: Partitioning, child: SparkPlan): Boolean = { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableColumnarShuffleSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableColumnarShuffleSuite.scala index 245b52d37109..c5f67f45d577 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableColumnarShuffleSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableColumnarShuffleSuite.scala @@ -17,7 +17,7 @@ package org.apache.gluten.execution import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.optimizer.BuildLeft +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} class GlutenClickHouseTPCHNullableColumnarShuffleSuite extends GlutenClickHouseTPCHAbstractSuite { @@ -63,7 +63,11 @@ class GlutenClickHouseTPCHNullableColumnarShuffleSuite extends GlutenClickHouseT val shjBuildLeft = df.queryExecution.executedPlan.collect { case shj: ShuffledHashJoinExecTransformerBase if shj.joinBuildSide == BuildLeft => shj } - assert(shjBuildLeft.size == 2) + assert(shjBuildLeft.size == 1) + val shjBuildRight = df.queryExecution.executedPlan.collect { + case shj: ShuffledHashJoinExecTransformerBase if shj.joinBuildSide == BuildRight => shj + } + assert(shjBuildRight.size == 1) } } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala index 0eb4de74209b..7f62c6993157 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala @@ -20,7 +20,7 @@ import org.apache.gluten.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.Alias -import org.apache.spark.sql.catalyst.optimizer.BuildLeft +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} class GlutenClickHouseTPCHNullableSuite extends GlutenClickHouseTPCHAbstractSuite { @@ -66,7 +66,11 @@ class GlutenClickHouseTPCHNullableSuite extends GlutenClickHouseTPCHAbstractSuit val shjBuildLeft = df.queryExecution.executedPlan.collect { case shj: ShuffledHashJoinExecTransformerBase if shj.joinBuildSide == BuildLeft => shj } - assert(shjBuildLeft.size == 2) + assert(shjBuildLeft.size == 1) + val shjBuildRight = df.queryExecution.executedPlan.collect { + case shj: ShuffledHashJoinExecTransformerBase if shj.joinBuildSide == BuildRight => shj + } + assert(shjBuildRight.size == 1) } } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala index 96b2cb09b163..d26891ddb1ea 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSuite.scala @@ -18,7 +18,7 @@ package org.apache.gluten.execution import org.apache.spark.SparkConf import org.apache.spark.sql.{Row, TestUtils} -import org.apache.spark.sql.catalyst.optimizer.BuildLeft +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight} import org.apache.spark.sql.types.{DecimalType, StructType} // Some sqls' line length exceeds 100 @@ -73,7 +73,11 @@ class GlutenClickHouseTPCHSuite extends GlutenClickHouseTPCHAbstractSuite { val shjBuildLeft = df.queryExecution.executedPlan.collect { case shj: ShuffledHashJoinExecTransformerBase if shj.joinBuildSide == BuildLeft => shj } - assert(shjBuildLeft.size == 2) + assert(shjBuildLeft.size == 1) + val shjBuildRight = df.queryExecution.executedPlan.collect { + case shj: ShuffledHashJoinExecTransformerBase if shj.joinBuildSide == BuildRight => shj + } + assert(shjBuildRight.size == 1) } } } diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index 6ebe36be3494..a1f1bb51cb98 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (110) +AdaptiveSparkPlan (112) +- == Final Plan == VeloxColumnarToRowExec (73) +- ^ SortExecTransformer (71) @@ -59,42 +59,44 @@ AdaptiveSparkPlan (110) +- ^ NoopFilter (56) +- ^ Scan parquet (55) +- == Initial Plan == - Sort (109) - +- Exchange (108) - +- Project (107) - +- BroadcastHashJoin Inner BuildRight (106) - :- Project (101) - : +- ShuffledHashJoin LeftSemi BuildRight (100) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Exchange (99) - : +- Project (98) - : +- BroadcastHashJoin Inner BuildLeft (97) - : :- BroadcastExchange (84) - : : +- BroadcastHashJoin LeftSemi BuildRight (83) - : : :- Filter (78) - : : : +- Scan parquet (77) - : : +- BroadcastExchange (82) - : : +- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- Filter (96) - : +- HashAggregate (95) - : +- Exchange (94) - : +- HashAggregate (93) - : +- BroadcastHashJoin LeftSemi BuildRight (92) - : :- Project (87) - : : +- Filter (86) - : : +- Scan parquet (85) - : +- BroadcastExchange (91) - : +- Project (90) - : +- Filter (89) - : +- Scan parquet (88) - +- BroadcastExchange (105) - +- Project (104) - +- Filter (103) - +- Scan parquet (102) + Sort (111) + +- Exchange (110) + +- Project (109) + +- BroadcastHashJoin Inner BuildRight (108) + :- Project (103) + : +- SortMergeJoin LeftSemi (102) + : :- Sort (77) + : : +- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Sort (101) + : +- Exchange (100) + : +- Project (99) + : +- BroadcastHashJoin Inner BuildLeft (98) + : :- BroadcastExchange (85) + : : +- BroadcastHashJoin LeftSemi BuildRight (84) + : : :- Filter (79) + : : : +- Scan parquet (78) + : : +- BroadcastExchange (83) + : : +- Project (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- Filter (97) + : +- HashAggregate (96) + : +- Exchange (95) + : +- HashAggregate (94) + : +- BroadcastHashJoin LeftSemi BuildRight (93) + : :- Project (88) + : : +- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (92) + : +- Project (91) + : +- Filter (90) + : +- Scan parquet (89) + +- BroadcastExchange (107) + +- Project (106) + +- Filter (105) + +- Scan parquet (104) (1) Scan parquet @@ -412,164 +414,172 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(77) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(79) Scan parquet +(80) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(81) Project +(82) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(82) BroadcastExchange +(83) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(84) BroadcastExchange +(85) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(85) Scan parquet +(86) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(87) Project +(88) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(90) Project +(91) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(91) BroadcastExchange +(92) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(93) HashAggregate +(94) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Filter +(97) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(98) Project +(99) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(99) Exchange +(100) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(101) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(102) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(101) Project +(103) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(102) Scan parquet +(104) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(103) Filter +(105) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(104) Project +(106) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(105) BroadcastExchange +(107) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(107) Project +(109) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(108) Exchange +(110) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(111) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(110) AdaptiveSparkPlan +(112) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index 27e59afbb7fc..4a899ae239be 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (109) +- == Final Plan == VeloxColumnarToRowExec (70) +- AQEShuffleRead (69) @@ -58,42 +58,44 @@ AdaptiveSparkPlan (107) +- ^ NoopFilter (56) +- ^ Scan parquet (55) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (108) + +- Exchange (107) + +- Project (106) + +- BroadcastHashJoin Inner BuildRight (105) + :- Project (100) + : +- SortMergeJoin LeftSemi (99) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (98) + : +- Exchange (97) + : +- Project (96) + : +- BroadcastHashJoin Inner BuildLeft (95) + : :- BroadcastExchange (82) + : : +- BroadcastHashJoin LeftSemi BuildRight (81) + : : :- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (80) + : : +- Project (79) + : : +- Filter (78) + : : +- Scan parquet (77) + : +- Filter (94) + : +- HashAggregate (93) + : +- Exchange (92) + : +- HashAggregate (91) + : +- BroadcastHashJoin LeftSemi BuildRight (90) + : :- Project (85) + : : +- Filter (84) + : : +- Scan parquet (83) + : +- BroadcastExchange (89) + : +- Project (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (104) + +- Project (103) + +- Filter (102) + +- Scan parquet (101) (1) Scan parquet @@ -401,164 +403,172 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(77) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(79) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(83) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(85) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(90) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(91) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(92) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(93) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(94) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(96) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(97) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(98) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(99) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(100) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(101) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(102) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(103) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(104) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(105) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(106) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(107) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(108) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(109) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index 7b494469aacc..9e03d8319537 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (110) +AdaptiveSparkPlan (112) +- == Final Plan == VeloxColumnarToRowExec (73) +- ^ SortExecTransformer (71) @@ -59,42 +59,44 @@ AdaptiveSparkPlan (110) +- ^ FilterExecTransformer (56) +- ^ Scan parquet (55) +- == Initial Plan == - Sort (109) - +- Exchange (108) - +- Project (107) - +- BroadcastHashJoin Inner BuildRight (106) - :- Project (101) - : +- ShuffledHashJoin LeftSemi BuildRight (100) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Exchange (99) - : +- Project (98) - : +- BroadcastHashJoin Inner BuildLeft (97) - : :- BroadcastExchange (84) - : : +- BroadcastHashJoin LeftSemi BuildRight (83) - : : :- Filter (78) - : : : +- Scan parquet (77) - : : +- BroadcastExchange (82) - : : +- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- Filter (96) - : +- HashAggregate (95) - : +- Exchange (94) - : +- HashAggregate (93) - : +- BroadcastHashJoin LeftSemi BuildRight (92) - : :- Project (87) - : : +- Filter (86) - : : +- Scan parquet (85) - : +- BroadcastExchange (91) - : +- Project (90) - : +- Filter (89) - : +- Scan parquet (88) - +- BroadcastExchange (105) - +- Project (104) - +- Filter (103) - +- Scan parquet (102) + Sort (111) + +- Exchange (110) + +- Project (109) + +- BroadcastHashJoin Inner BuildRight (108) + :- Project (103) + : +- SortMergeJoin LeftSemi (102) + : :- Sort (77) + : : +- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Sort (101) + : +- Exchange (100) + : +- Project (99) + : +- BroadcastHashJoin Inner BuildLeft (98) + : :- BroadcastExchange (85) + : : +- BroadcastHashJoin LeftSemi BuildRight (84) + : : :- Filter (79) + : : : +- Scan parquet (78) + : : +- BroadcastExchange (83) + : : +- Project (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- Filter (97) + : +- HashAggregate (96) + : +- Exchange (95) + : +- HashAggregate (94) + : +- BroadcastHashJoin LeftSemi BuildRight (93) + : :- Project (88) + : : +- Filter (87) + : : +- Scan parquet (86) + : +- BroadcastExchange (92) + : +- Project (91) + : +- Filter (90) + : +- Scan parquet (89) + +- BroadcastExchange (107) + +- Project (106) + +- Filter (105) + +- Scan parquet (104) (1) Scan parquet @@ -412,164 +414,172 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(77) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(78) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(78) Filter +(79) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(79) Scan parquet +(80) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(81) Project +(82) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(82) BroadcastExchange +(83) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(84) BroadcastExchange +(85) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(85) Scan parquet +(86) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(87) Project +(88) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(90) Project +(91) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(91) BroadcastExchange +(92) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(93) HashAggregate +(94) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Filter +(97) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(97) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(98) Project +(99) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(99) Exchange +(100) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(101) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(102) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(101) Project +(103) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(102) Scan parquet +(104) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(103) Filter +(105) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(104) Project +(106) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(105) BroadcastExchange +(107) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(106) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(107) Project +(109) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(108) Exchange +(110) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(111) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(110) AdaptiveSparkPlan +(112) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index a0edead7013d..5cd3c9d35c2c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (109) +- == Final Plan == VeloxColumnarToRowExec (70) +- AQEShuffleRead (69) @@ -58,42 +58,44 @@ AdaptiveSparkPlan (107) +- ^ FilterExecTransformer (56) +- ^ Scan parquet (55) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (108) + +- Exchange (107) + +- Project (106) + +- BroadcastHashJoin Inner BuildRight (105) + :- Project (100) + : +- SortMergeJoin LeftSemi (99) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (98) + : +- Exchange (97) + : +- Project (96) + : +- BroadcastHashJoin Inner BuildLeft (95) + : :- BroadcastExchange (82) + : : +- BroadcastHashJoin LeftSemi BuildRight (81) + : : :- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (80) + : : +- Project (79) + : : +- Filter (78) + : : +- Scan parquet (77) + : +- Filter (94) + : +- HashAggregate (93) + : +- Exchange (92) + : +- HashAggregate (91) + : +- BroadcastHashJoin LeftSemi BuildRight (90) + : :- Project (85) + : : +- Filter (84) + : : +- Scan parquet (83) + : +- BroadcastExchange (89) + : +- Project (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (104) + +- Project (103) + +- Filter (102) + +- Scan parquet (101) (1) Scan parquet @@ -401,164 +403,172 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(77) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(79) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(82) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(83) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(85) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(90) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(91) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(92) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(93) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(94) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(96) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(97) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(98) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(99) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(100) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(101) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(102) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(103) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(104) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(105) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(106) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(107) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(108) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(109) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 85176d8c6011..ec46bfd07b91 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == VeloxColumnarToRowExec (67) +- TakeOrderedAndProjectExecTransformer (66) @@ -54,32 +54,38 @@ AdaptiveSparkPlan (94) +- ^ NoopFilter (45) +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- ShuffledHashJoin Inner BuildRight (88) - :- Exchange (84) - : +- Project (83) - : +- ShuffledHashJoin Inner BuildRight (82) - : :- Exchange (77) - : : +- Project (76) - : : +- ShuffledHashJoin Inner BuildRight (75) - : : :- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (81) - : +- Project (80) - : +- Filter (79) - : +- Scan parquet (78) - +- Exchange (87) - +- Filter (86) - +- Scan parquet (85) + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -371,116 +377,140 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(74) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(75) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(76) Project +(78) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(77) Exchange +(79) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(79) Filter +(82) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(80) Project +(83) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(81) Exchange +(84) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Project +(87) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(84) Exchange +(88) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(87) Exchange +(92) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) ShuffledHashJoin +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(95) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(90) HashAggregate +(96) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(91) Exchange +(97) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(98) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) TakeOrderedAndProject +(99) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 7c749c0a5ec6..cccf1408bea9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (78) +AdaptiveSparkPlan (82) +- == Final Plan == VeloxColumnarToRowExec (56) +- ^ SortExecTransformer (54) @@ -45,27 +45,31 @@ AdaptiveSparkPlan (78) +- ^ NoopFilter (28) +- ^ Scan parquet (27) +- == Initial Plan == - Sort (77) - +- Exchange (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Project (71) - +- ShuffledHashJoin Inner BuildRight (70) - :- Exchange (65) - : +- Project (64) - : +- ShuffledHashJoin Inner BuildRight (63) - : :- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Exchange (62) - : +- Filter (61) - : +- Scan parquet (60) - +- Exchange (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) (1) Scan parquet @@ -311,92 +315,108 @@ Condition : isnotnull(ps_suppkey#X) Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) Exchange +(63) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) ShuffledHashJoin +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(64) Project +(66) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(67) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Scan parquet +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(68) Project +(71) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(69) Exchange +(72) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) ShuffledHashJoin +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) Project +(75) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(72) HashAggregate +(76) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(73) Exchange +(77) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(78) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(75) Filter +(79) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(76) Exchange +(80) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Sort +(81) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(78) AdaptiveSparkPlan +(82) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index 5cf27c6e0cb9..17cdf62608cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,20 +31,22 @@ AdaptiveSparkPlan (53) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin Inner BuildLeft (46) - :- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -218,60 +220,68 @@ Condition : isnotnull(o_orderkey#X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(44) Project +(45) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(45) Exchange +(46) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(49) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(51) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(53) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index d3904d8d079e..730f0e0a438a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (58) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ SortExecTransformer (38) @@ -33,21 +33,23 @@ AdaptiveSparkPlan (56) +- ^ NoopFilter (10) +- ^ Scan parquet (9) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftOuter BuildRight (47) - :- Exchange (42) - : +- Scan parquet (41) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftOuter (49) + :- Sort (43) + : +- Exchange (42) + : +- Scan parquet (41) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -229,74 +231,82 @@ ReadSchema: struct Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(45) Project +(46) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(46) Exchange +(47) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(50) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(51) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(53) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(54) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(55) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(56) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(57) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index 00b5fb4142f3..55111a31f874 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (39) +- == Final Plan == VeloxColumnarToRowExec (25) +- ^ ProjectExecTransformer (23) @@ -22,17 +22,19 @@ AdaptiveSparkPlan (37) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- ShuffledHashJoin Inner BuildRight (33) - :- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -162,44 +164,52 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Scan parquet +(30) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(31) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(32) Exchange +(33) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) ShuffledHashJoin +(34) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) Project +(36) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(36) HashAggregate +(38) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(37) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index eab0e2908a10..db2df6c87544 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) @@ -28,20 +28,22 @@ AdaptiveSparkPlan (48) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- Project (45) - +- ShuffledHashJoin Inner BuildLeft (44) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Filter (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- Project (47) + +- SortMergeJoin Inner (46) + :- Sort (37) + : +- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Sort (45) + +- Filter (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -197,60 +199,68 @@ Condition : isnotnull(s_suppkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(37) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(38) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(39) Project +(40) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(43) Filter +(44) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(44) ShuffledHashJoin +(45) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(46) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(45) Project +(47) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(46) Exchange +(48) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 354bd4f3fabd..2eb5668906ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (71) +- == Final Plan == VeloxColumnarToRowExec (47) +- ^ SortExecTransformer (45) @@ -38,27 +38,29 @@ AdaptiveSparkPlan (69) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- ShuffledHashJoin Inner BuildRight (59) - :- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (58) - +- Filter (57) - +- Scan parquet (56) + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -296,74 +298,82 @@ Join condition: None Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(58) Exchange +(59) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) ShuffledHashJoin +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(60) Project +(62) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(61) HashAggregate +(63) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(65) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(67) Exchange +(69) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(70) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 848d4e2ce4f8..5226aacff753 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (63) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ ProjectExecTransformer (38) @@ -35,25 +35,28 @@ AdaptiveSparkPlan (60) +- ^ NoopFilter (22) +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (59) - +- HashAggregate (58) - +- Project (57) - +- ShuffledHashJoin Inner BuildRight (56) - :- Project (49) - : +- ShuffledHashJoin Inner BuildRight (48) - : :- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Filter (51) - +- Scan parquet (50) + HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (44) + : : +- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -247,90 +250,102 @@ Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(44) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(45) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(46) Project +(47) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(47) Exchange +(48) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(49) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(49) Project +(51) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(50) Scan parquet +(52) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(51) Filter +(53) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(52) HashAggregate +(54) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(53) Exchange +(55) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) Filter +(57) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(56) ShuffledHashJoin +(58) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(57) Project +(60) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(58) HashAggregate +(61) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(59) HashAggregate +(62) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(60) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 3d4743403809..c1287b2d685a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (110) +- == Final Plan == VeloxColumnarToRowExec (70) +- TakeOrderedAndProjectExecTransformer (69) @@ -58,38 +58,45 @@ AdaptiveSparkPlan (103) +- ShuffleQueryStage (57) +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (109) + +- HashAggregate (108) + +- HashAggregate (107) + +- Project (106) + +- SortMergeJoin Inner (105) + :- Sort (92) + : +- Exchange (91) + : +- Project (90) + : +- SortMergeJoin Inner (89) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (88) + : +- Exchange (87) + : +- SortMergeJoin LeftSemi (86) + : :- Sort (78) + : : +- Exchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Sort (85) + : +- Project (84) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- Scan parquet (79) + +- SortMergeJoin LeftSemi (104) + :- Sort (96) + : +- Exchange (95) + : +- Filter (94) + : +- Scan parquet (93) + +- Sort (103) + +- Project (102) + +- Filter (101) + +- HashAggregate (100) + +- Exchange (99) + +- HashAggregate (98) + +- Scan parquet (97) (1) Scan parquet @@ -401,154 +408,182 @@ Condition : isnotnull(c_custkey#X) Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(77) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(80) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(81) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(82) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(83) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(84) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(85) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(84) Exchange +(87) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(88) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(89) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(86) Project +(90) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(91) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(92) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(93) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(98) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(99) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(100) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(101) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(102) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(103) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) ShuffledHashJoin +(105) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(99) Project +(106) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(107) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(108) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(109) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index 6ec9ae965ee9..21e4f472f3b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (36) +AdaptiveSparkPlan (38) +- == Final Plan == VeloxColumnarToRowExec (24) +- ^ RegularHashAggregateExecTransformer (22) @@ -21,17 +21,19 @@ AdaptiveSparkPlan (36) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (35) - +- HashAggregate (34) - +- Project (33) - +- ShuffledHashJoin Inner BuildRight (32) - :- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -157,44 +159,52 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(29) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(30) Filter +(31) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(31) Exchange +(32) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) ShuffledHashJoin +(33) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(33) Project +(35) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(34) HashAggregate +(36) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(36) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 24be4842e1b8..1ac0992834eb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (136) +AdaptiveSparkPlan (146) +- == Final Plan == VeloxColumnarToRowExec (96) +- ^ SortExecTransformer (94) @@ -76,45 +76,55 @@ AdaptiveSparkPlan (136) +- ^ NoopFilter (78) +- ^ Scan parquet (77) +- == Initial Plan == - Sort (135) - +- Exchange (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin LeftSemi BuildRight (125) - : :- Exchange (99) - : : +- Filter (98) - : : +- Scan parquet (97) - : +- Exchange (124) - : +- Project (123) - : +- ShuffledHashJoin Inner BuildLeft (122) - : :- Exchange (108) - : : +- ShuffledHashJoin LeftSemi BuildRight (107) - : : :- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (106) - : : +- Project (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (121) - : +- Filter (120) - : +- HashAggregate (119) - : +- HashAggregate (118) - : +- ShuffledHashJoin LeftSemi BuildRight (117) - : :- Exchange (112) - : : +- Project (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- Exchange (116) - : +- Project (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Exchange (131) - +- Project (130) - +- Filter (129) - +- Scan parquet (128) + Sort (145) + +- Exchange (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (136) + : +- Exchange (135) + : +- Project (134) + : +- SortMergeJoin LeftSemi (133) + : :- Sort (100) + : : +- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Sort (132) + : +- Exchange (131) + : +- Project (130) + : +- SortMergeJoin Inner (129) + : :- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftSemi (110) + : : :- Sort (104) + : : : +- Exchange (103) + : : : +- Filter (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (128) + : +- Exchange (127) + : +- Filter (126) + : +- HashAggregate (125) + : +- HashAggregate (124) + : +- SortMergeJoin LeftSemi (123) + : :- Sort (117) + : : +- Exchange (116) + : : +- Project (115) + : : +- Filter (114) + : : +- Scan parquet (113) + : +- Sort (122) + : +- Exchange (121) + : +- Project (120) + : +- Filter (119) + : +- Scan parquet (118) + +- Sort (141) + +- Exchange (140) + +- Project (139) + +- Filter (138) + +- Scan parquet (137) (1) Scan parquet @@ -518,176 +528,216 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(100) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(101) Filter +(102) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(102) Exchange +(103) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(104) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(105) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(107) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(108) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(109) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) Exchange +(111) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(112) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(113) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(110) Filter +(114) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(111) Project +(115) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(112) Exchange +(116) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(117) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(118) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(115) Project +(120) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(116) Exchange +(121) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(118) HashAggregate +(124) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(119) HashAggregate +(125) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(120) Filter +(126) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(121) Exchange +(127) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(128) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(129) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(123) Project +(130) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(124) Exchange +(131) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(132) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(133) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(126) Project +(134) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(127) Exchange +(135) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(137) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(129) Filter +(138) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(130) Project +(139) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(131) Exchange +(140) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(143) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(134) Exchange +(144) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Sort +(145) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(136) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index b8c363fce329..e2a72528c4ed 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (138) +- == Final Plan == VeloxColumnarToRowExec (92) +- TakeOrderedAndProjectExecTransformer (91) @@ -73,42 +73,51 @@ AdaptiveSparkPlan (129) +- ^ NoopFilter (71) +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- ShuffledHashJoin Inner BuildRight (123) - :- Exchange (118) - : +- Project (117) - : +- ShuffledHashJoin Inner BuildRight (116) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildLeft (109) - : : :- Exchange (95) - : : : +- Filter (94) - : : : +- Scan parquet (93) - : : +- Exchange (108) - : : +- ShuffledHashJoin LeftAnti BuildRight (107) - : : :- ShuffledHashJoin LeftSemi BuildRight (102) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- Filter (97) - : : : : +- Scan parquet (96) - : : : +- Exchange (101) - : : : +- Scan parquet (100) - : : +- Exchange (106) - : : +- Project (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (115) - : +- Project (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + TakeOrderedAndProject (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (117) + : : +- Exchange (116) + : : +- Project (115) + : : +- SortMergeJoin Inner (114) + : : :- Sort (96) + : : : +- Exchange (95) + : : : +- Filter (94) + : : : +- Scan parquet (93) + : : +- Sort (113) + : : +- Exchange (112) + : : +- SortMergeJoin LeftAnti (111) + : : :- SortMergeJoin LeftSemi (105) + : : : :- Sort (101) + : : : : +- Exchange (100) + : : : : +- Project (99) + : : : : +- Filter (98) + : : : : +- Scan parquet (97) + : : : +- Sort (104) + : : : +- Exchange (103) + : : : +- Scan parquet (102) + : : +- Sort (110) + : : +- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Sort (122) + : +- Exchange (121) + : +- Project (120) + : +- Filter (119) + : +- Scan parquet (118) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) (1) Scan parquet @@ -501,163 +510,199 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(96) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(97) Filter +(98) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(98) Project +(99) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(99) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(101) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(102) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(101) Exchange +(103) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(104) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(105) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(103) Scan parquet +(106) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(104) Filter +(107) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(105) Project +(108) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(106) Exchange +(109) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(110) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(111) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(108) Exchange +(112) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(113) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(114) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(110) Project +(115) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(111) Exchange +(116) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(117) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(118) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(113) Filter +(119) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(114) Project +(120) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(115) Exchange +(121) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(122) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(117) Project +(124) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(118) Exchange +(125) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Scan parquet +(126) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(120) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(121) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(122) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) ShuffledHashJoin +(131) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(124) Project +(133) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(125) HashAggregate +(134) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(126) Exchange +(135) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(136) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(128) TakeOrderedAndProject +(137) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(129) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index f336a73676ea..984abd470378 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (52) +- == Final Plan == VeloxColumnarToRowExec (37) +- ^ SortExecTransformer (35) @@ -30,18 +30,20 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftAnti BuildRight (43) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (42) - +- Scan parquet (41) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) (1) Scan parquet @@ -210,51 +212,59 @@ Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23 Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(42) Exchange +(43) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(48) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(50) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(51) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index f188fa96b0d8..58484edaa685 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == VeloxColumnarToRowExec (43) +- TakeOrderedAndProjectExecTransformer (42) @@ -36,25 +36,29 @@ AdaptiveSparkPlan (63) +- ^ NoopFilter (28) +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- ShuffledHashJoin Inner BuildRight (58) - :- Exchange (53) - : +- Project (52) - : +- ShuffledHashJoin Inner BuildLeft (51) - : :- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Project (63) + +- SortMergeJoin Inner (62) + :- Sort (56) + : +- Exchange (55) + : +- Project (54) + : +- SortMergeJoin Inner (53) + : :- Sort (48) + : : +- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Sort (52) + : +- Exchange (51) + : +- Filter (50) + : +- Scan parquet (49) + +- Sort (61) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -256,80 +260,96 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Scan parquet +(48) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(49) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(49) Filter +(50) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(50) Exchange +(51) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(52) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(53) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(52) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(53) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(56) Project +(59) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(57) Exchange +(60) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) ShuffledHashJoin +(61) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(62) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) Project +(63) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(60) HashAggregate +(64) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(61) HashAggregate +(65) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 42a8fef3563f..cb7a3c3a0955 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (56) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,21 +31,23 @@ AdaptiveSparkPlan (54) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftSemi BuildRight (47) - :- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -223,60 +225,68 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(45) Project +(46) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(46) Exchange +(47) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(50) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(51) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(54) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(55) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index 378085655899..930a5a0bf488 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (156) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,45 +83,55 @@ AdaptiveSparkPlan (146) +- ^ NoopFilter (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- HashAggregate (143) - +- Exchange (142) - +- HashAggregate (141) - +- Project (140) - +- ShuffledHashJoin Inner BuildRight (139) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Project (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (138) - +- Project (137) - +- Filter (136) - +- Scan parquet (135) + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -567,176 +577,216 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(113) Exchange +(114) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(115) Project +(117) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(118) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(119) Exchange +(122) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(121) Project +(125) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(122) Exchange +(126) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(125) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(127) Project +(133) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(128) Exchange +(134) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(131) Exchange +(138) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(141) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(134) Exchange +(142) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(137) Project +(146) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(138) Exchange +(147) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(140) Project +(150) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(141) HashAggregate +(151) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(142) Exchange +(152) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) HashAggregate +(153) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(144) Exchange +(154) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(155) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(146) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index a7054770a17e..d9eb23cb737e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (139) +AdaptiveSparkPlan (149) +- == Final Plan == VeloxColumnarToRowExec (101) +- ^ SortExecTransformer (99) @@ -79,43 +79,53 @@ AdaptiveSparkPlan (139) +- ShuffleQueryStage (79) +- ReusedExchange (78) +- == Initial Plan == - Sort (138) - +- Exchange (137) - +- HashAggregate (136) - +- Exchange (135) - +- HashAggregate (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (128) - : +- Project (127) - : +- ShuffledHashJoin Inner BuildRight (126) - : :- Exchange (122) - : : +- Project (121) - : : +- ShuffledHashJoin Inner BuildRight (120) - : : :- Exchange (116) - : : : +- Project (115) - : : : +- ShuffledHashJoin Inner BuildRight (114) - : : : :- Exchange (110) - : : : : +- Project (109) - : : : : +- ShuffledHashJoin Inner BuildLeft (108) - : : : : :- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Exchange (113) - : : : +- Filter (112) - : : : +- Scan parquet (111) - : : +- Exchange (119) - : : +- Filter (118) - : : +- Scan parquet (117) - : +- Exchange (125) - : +- Filter (124) - : +- Scan parquet (123) - +- Exchange (131) - +- Filter (130) - +- Scan parquet (129) + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) (1) Scan parquet @@ -537,168 +547,208 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(107) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(108) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(111) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(110) Exchange +(112) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(112) Filter +(115) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(113) Exchange +(116) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(119) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(120) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(118) Filter +(123) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(119) Exchange +(124) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(121) Project +(127) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(122) Exchange +(128) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(124) Filter +(131) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(125) Exchange +(132) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(135) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(128) Exchange +(136) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(130) Filter +(139) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(131) Exchange +(140) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(133) Project +(143) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(134) HashAggregate +(144) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(135) Exchange +(145) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) HashAggregate +(146) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(137) Exchange +(147) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(148) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(139) AdaptiveSparkPlan +(149) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index cdede8445908..5c9e51b95c60 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (193) +AdaptiveSparkPlan (207) +- == Final Plan == VeloxColumnarToRowExec (141) +- ^ SortExecTransformer (139) @@ -110,57 +110,71 @@ AdaptiveSparkPlan (193) +- ^ NoopFilter (113) +- ^ Scan parquet (112) +- == Initial Plan == - Sort (192) - +- Exchange (191) - +- HashAggregate (190) - +- Exchange (189) - +- HashAggregate (188) - +- Project (187) - +- ShuffledHashJoin Inner BuildRight (186) - :- Exchange (181) - : +- Project (180) - : +- ShuffledHashJoin Inner BuildRight (179) - : :- Exchange (175) - : : +- Project (174) - : : +- ShuffledHashJoin Inner BuildRight (173) - : : :- Exchange (169) - : : : +- Project (168) - : : : +- ShuffledHashJoin Inner BuildRight (167) - : : : :- Exchange (163) - : : : : +- Project (162) - : : : : +- ShuffledHashJoin Inner BuildRight (161) - : : : : :- Exchange (157) - : : : : : +- Project (156) - : : : : : +- ShuffledHashJoin Inner BuildRight (155) - : : : : : :- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) - : : : : : : :- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Exchange (148) - : : : : : : +- Filter (147) - : : : : : : +- Scan parquet (146) - : : : : : +- Exchange (154) - : : : : : +- Filter (153) - : : : : : +- Scan parquet (152) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Exchange (166) - : : : +- Filter (165) - : : : +- Scan parquet (164) - : : +- Exchange (172) - : : +- Filter (171) - : : +- Scan parquet (170) - : +- Exchange (178) - : +- Filter (177) - : +- Scan parquet (176) - +- Exchange (185) - +- Project (184) - +- Filter (183) - +- Scan parquet (182) + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) (1) Scan parquet @@ -750,228 +764,284 @@ Input [2]: [p_partkey#X, p_type#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(147) Filter +(148) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(148) Exchange +(149) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(150) Project +(152) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(153) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(153) Filter +(156) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(154) Exchange +(157) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(156) Project +(160) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(157) Exchange +(161) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(159) Filter +(164) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(160) Exchange +(165) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(161) ShuffledHashJoin +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(162) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(163) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Scan parquet +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(165) Filter +(172) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(166) Exchange +(173) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(167) ShuffledHashJoin +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(168) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(169) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) Scan parquet +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(171) Filter +(180) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(172) Exchange +(181) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(173) ShuffledHashJoin +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(174) Project +(184) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(175) Exchange +(185) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Scan parquet +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(177) Filter +(188) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(178) Exchange +(189) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(179) ShuffledHashJoin +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(180) Project +(192) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(181) Exchange +(193) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(182) Scan parquet +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(183) Filter +(196) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(184) Project +(197) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(185) Exchange +(198) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(186) ShuffledHashJoin +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(187) Project +(201) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(188) HashAggregate +(202) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(189) Exchange +(203) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(190) HashAggregate +(204) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(191) Exchange +(205) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(206) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(193) AdaptiveSparkPlan +(207) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 11a02d0a54d2..2abb6ec215c6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (155) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,44 +83,54 @@ AdaptiveSparkPlan (145) +- ^ NoopFilter (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (137) - +- Filter (136) - +- Scan parquet (135) + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -570,168 +580,208 @@ Input [2]: [p_partkey#X, p_name#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(113) Exchange +(114) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(115) Project +(117) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(118) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(119) Exchange +(122) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(121) Project +(125) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(122) Exchange +(126) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(125) Exchange +(130) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(127) Project +(133) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(128) Exchange +(134) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(131) Exchange +(138) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(133) Project +(141) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(134) Exchange +(142) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(137) Exchange +(146) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(139) Project +(149) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(140) HashAggregate +(150) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(141) Exchange +(151) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(152) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(143) Exchange +(153) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(154) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(155) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 5a74265ab590..3be5f1996fa8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == VeloxColumnarToRowExec (67) +- TakeOrderedAndProjectExecTransformer (66) @@ -54,32 +54,38 @@ AdaptiveSparkPlan (94) +- ^ NoopFilter (45) +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- ShuffledHashJoin Inner BuildRight (88) - :- Exchange (84) - : +- Project (83) - : +- ShuffledHashJoin Inner BuildRight (82) - : :- Exchange (77) - : : +- Project (76) - : : +- ShuffledHashJoin Inner BuildRight (75) - : : :- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (81) - : +- Project (80) - : +- Filter (79) - : +- Scan parquet (78) - +- Exchange (87) - +- Filter (86) - +- Scan parquet (85) + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -371,116 +377,140 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(74) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(75) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(76) Project +(78) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(77) Exchange +(79) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(79) Filter +(82) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(80) Project +(83) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(81) Exchange +(84) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Project +(87) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(84) Exchange +(88) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(87) Exchange +(92) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) ShuffledHashJoin +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(95) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(90) HashAggregate +(96) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(91) Exchange +(97) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(98) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) TakeOrderedAndProject +(99) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 8d17beb8c0a9..2347eb9b677e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (78) +AdaptiveSparkPlan (82) +- == Final Plan == VeloxColumnarToRowExec (56) +- ^ SortExecTransformer (54) @@ -45,27 +45,31 @@ AdaptiveSparkPlan (78) +- ^ NoopFilter (28) +- ^ Scan parquet (27) +- == Initial Plan == - Sort (77) - +- Exchange (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Project (71) - +- ShuffledHashJoin Inner BuildRight (70) - :- Exchange (65) - : +- Project (64) - : +- ShuffledHashJoin Inner BuildRight (63) - : :- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Exchange (62) - : +- Filter (61) - : +- Scan parquet (60) - +- Exchange (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) (1) Scan parquet @@ -311,359 +315,395 @@ Condition : isnotnull(ps_suppkey#X) Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) Exchange +(63) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) ShuffledHashJoin +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(64) Project +(66) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(67) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Scan parquet +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(68) Project +(71) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(69) Exchange +(72) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) ShuffledHashJoin +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) Project +(75) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(72) HashAggregate +(76) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(73) Exchange +(77) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(78) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(75) Filter +(79) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(76) Exchange +(80) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Sort +(81) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(78) AdaptiveSparkPlan +(82) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (110) - +- ^ ProjectExecTransformer (108) - +- ^ RegularHashAggregateExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ ProjectExecTransformer (105) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - :- ^ InputIteratorTransformer (99) - : +- ShuffleQueryStage (97), Statistics(X) - : +- ColumnarExchange (96) - : +- VeloxAppendBatches (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (87) - : : +- ShuffleQueryStage (85), Statistics(X) - : : +- ColumnarExchange (84) - : : +- VeloxAppendBatches (83) - : : +- ^ ProjectExecTransformer (81) - : : +- ^ NoopFilter (80) - : : +- ^ Scan parquet (79) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ReusedExchange (88) - +- ^ InputIteratorTransformer (103) - +- ShuffleQueryStage (101), Statistics(X) - +- ReusedExchange (100) + VeloxColumnarToRowExec (114) + +- ^ ProjectExecTransformer (112) + +- ^ RegularHashAggregateExecTransformer (111) + +- ^ RegularHashAggregateExecTransformer (110) + +- ^ ProjectExecTransformer (109) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) + :- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) + : :- ^ InputIteratorTransformer (91) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- ColumnarExchange (88) + : : +- VeloxAppendBatches (87) + : : +- ^ ProjectExecTransformer (85) + : : +- ^ NoopFilter (84) + : : +- ^ Scan parquet (83) + : +- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ReusedExchange (92) + +- ^ InputIteratorTransformer (107) + +- ShuffleQueryStage (105), Statistics(X) + +- ReusedExchange (104) +- == Initial Plan == - HashAggregate (127) - +- HashAggregate (126) - +- Project (125) - +- ShuffledHashJoin Inner BuildRight (124) - :- Exchange (119) - : +- Project (118) - : +- ShuffledHashJoin Inner BuildRight (117) - : :- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- Exchange (123) - +- Project (122) - +- Filter (121) - +- Scan parquet (120) - - -(79) Scan parquet + HashAggregate (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Filter (116) + : : +- Scan parquet (115) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) + + +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) NoopFilter +(84) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ProjectExecTransformer +(85) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(82) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(83) VeloxAppendBatches +(87) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(88) ReusedExchange [Reuses operator id: 15] +(92) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(94) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(95) VeloxAppendBatches +(99) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(96) ColumnarExchange +(100) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(97) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(98) InputAdapter +(102) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(100) ReusedExchange [Reuses operator id: 32] +(104) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(101) ShuffleQueryStage +(105) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(102) InputAdapter +(106) InputAdapter Input [1]: [n_nationkey#X] -(103) InputIteratorTransformer +(107) InputIteratorTransformer Input [1]: [n_nationkey#X] -(104) ShuffledHashJoinExecTransformer +(108) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(105) ProjectExecTransformer +(109) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(106) RegularHashAggregateExecTransformer +(110) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(107) RegularHashAggregateExecTransformer +(111) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(108) ProjectExecTransformer +(112) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(109) WholeStageCodegenTransformer (X) +(113) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(110) VeloxColumnarToRowExec +(114) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(111) Scan parquet +(115) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(112) Filter +(116) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(113) Exchange +(117) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) Scan parquet +(118) Sort +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(119) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(115) Filter +(120) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(116) Exchange +(121) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(118) Project +(124) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(119) Exchange +(125) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Scan parquet +(126) Sort +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(121) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(122) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(123) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) ShuffledHashJoin +(131) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(125) Project +(133) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(126) HashAggregate +(134) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(127) HashAggregate +(135) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(128) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index dd1259eb8876..b0f084e2d048 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,20 +31,22 @@ AdaptiveSparkPlan (53) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin Inner BuildLeft (46) - :- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -218,60 +220,68 @@ Condition : isnotnull(o_orderkey#X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(44) Project +(45) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(45) Exchange +(46) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(49) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(51) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(53) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index d43ad2a9c271..07c32ff95fb1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (58) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ SortExecTransformer (38) @@ -33,21 +33,23 @@ AdaptiveSparkPlan (56) +- ^ NoopFilter (10) +- ^ Scan parquet (9) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftOuter BuildRight (47) - :- Exchange (42) - : +- Scan parquet (41) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftOuter (49) + :- Sort (43) + : +- Exchange (42) + : +- Scan parquet (41) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -229,74 +231,82 @@ ReadSchema: struct Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(45) Project +(46) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(46) Exchange +(47) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(50) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(51) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(53) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(54) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(55) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(56) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(57) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index cb3ddbb3a2f7..11bbb2a71e79 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (39) +- == Final Plan == VeloxColumnarToRowExec (25) +- ^ ProjectExecTransformer (23) @@ -22,17 +22,19 @@ AdaptiveSparkPlan (37) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- ShuffledHashJoin Inner BuildRight (33) - :- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -162,44 +164,52 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Scan parquet +(30) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(31) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(32) Exchange +(33) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) ShuffledHashJoin +(34) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) Project +(36) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(36) HashAggregate +(38) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(37) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 60521d6c62cd..be97f58cf438 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == VeloxColumnarToRowExec (30) +- AQEShuffleRead (29) @@ -27,20 +27,22 @@ AdaptiveSparkPlan (45) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -186,221 +188,229 @@ Condition : isnotnull(s_suppkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(34) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(40) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(42) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (71) +AdaptiveSparkPlan (73) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ ProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- VeloxAppendBatches (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (64) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ ProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ ProjectExecTransformer (52) + +- ^ FlushableHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ NoopFilter (49) + +- ^ Scan parquet (48) +- == Initial Plan == - HashAggregate (70) - +- HashAggregate (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (72) + +- HashAggregate (71) + +- HashAggregate (70) + +- Exchange (69) + +- HashAggregate (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) -(46) Scan parquet +(48) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(47) NoopFilter +(49) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(48) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(49) FlushableHashAggregateExecTransformer +(51) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(52) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(52) VeloxAppendBatches +(54) VeloxAppendBatches Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) ColumnarExchange +(55) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(57) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) ProjectExecTransformer +(60) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(60) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(62) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(63) Scan parquet +(65) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(64) Filter +(66) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(65) Project +(67) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(67) Exchange +(69) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(70) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(69) HashAggregate +(71) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(70) HashAggregate +(72) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(71) AdaptiveSparkPlan +(73) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index 029516a40506..86d2f321f653 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (71) +- == Final Plan == VeloxColumnarToRowExec (47) +- ^ SortExecTransformer (45) @@ -38,27 +38,29 @@ AdaptiveSparkPlan (69) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- ShuffledHashJoin Inner BuildRight (59) - :- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (58) - +- Filter (57) - +- Scan parquet (56) + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -296,74 +298,82 @@ Join condition: None Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(58) Exchange +(59) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) ShuffledHashJoin +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(60) Project +(62) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(61) HashAggregate +(63) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(65) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(67) Exchange +(69) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(70) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index e2d1503799a9..6a2e47576cad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (63) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ ProjectExecTransformer (38) @@ -35,25 +35,28 @@ AdaptiveSparkPlan (60) +- ^ NoopFilter (22) +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (59) - +- HashAggregate (58) - +- Project (57) - +- ShuffledHashJoin Inner BuildRight (56) - :- Project (49) - : +- ShuffledHashJoin Inner BuildRight (48) - : :- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Filter (51) - +- Scan parquet (50) + HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (44) + : : +- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -247,90 +250,102 @@ Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(44) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(45) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(46) Project +(47) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(47) Exchange +(48) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(49) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(49) Project +(51) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(50) Scan parquet +(52) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(51) Filter +(53) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(52) HashAggregate +(54) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(53) Exchange +(55) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) Filter +(57) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(56) ShuffledHashJoin +(58) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(57) Project +(60) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(58) HashAggregate +(61) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(59) HashAggregate +(62) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(60) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index a0e052432bb3..7fe13a003017 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (110) +- == Final Plan == VeloxColumnarToRowExec (70) +- TakeOrderedAndProjectExecTransformer (69) @@ -58,38 +58,45 @@ AdaptiveSparkPlan (103) +- ShuffleQueryStage (57), Statistics(X) +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (109) + +- HashAggregate (108) + +- HashAggregate (107) + +- Project (106) + +- SortMergeJoin Inner (105) + :- Sort (92) + : +- Exchange (91) + : +- Project (90) + : +- SortMergeJoin Inner (89) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (88) + : +- Exchange (87) + : +- SortMergeJoin LeftSemi (86) + : :- Sort (78) + : : +- Exchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Sort (85) + : +- Project (84) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- Scan parquet (79) + +- SortMergeJoin LeftSemi (104) + :- Sort (96) + : +- Exchange (95) + : +- Filter (94) + : +- Scan parquet (93) + +- Sort (103) + +- Project (102) + +- Filter (101) + +- HashAggregate (100) + +- Exchange (99) + +- HashAggregate (98) + +- Scan parquet (97) (1) Scan parquet @@ -401,154 +408,182 @@ Condition : isnotnull(c_custkey#X) Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(77) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(80) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(81) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(82) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(83) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(84) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(85) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(84) Exchange +(87) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(88) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(89) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(86) Project +(90) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(91) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(92) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(93) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(98) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(99) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(100) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(101) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(102) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(103) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) ShuffledHashJoin +(105) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(99) Project +(106) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(107) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(108) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(109) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index 440383aa1cd7..34abb726b85a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (36) +AdaptiveSparkPlan (38) +- == Final Plan == VeloxColumnarToRowExec (24) +- ^ RegularHashAggregateExecTransformer (22) @@ -21,17 +21,19 @@ AdaptiveSparkPlan (36) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (35) - +- HashAggregate (34) - +- Project (33) - +- ShuffledHashJoin Inner BuildRight (32) - :- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -157,44 +159,52 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(29) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(30) Filter +(31) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(31) Exchange +(32) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) ShuffledHashJoin +(33) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(33) Project +(35) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(34) HashAggregate +(36) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(36) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index d3cff30e0ce2..2cf50b2a3a98 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (143) +- == Final Plan == VeloxColumnarToRowExec (93) +- AQEShuffleRead (92) @@ -75,45 +75,55 @@ AdaptiveSparkPlan (133) +- ^ NoopFilter (78) +- ^ Scan parquet (77) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- Project (130) - +- ShuffledHashJoin Inner BuildRight (129) - :- Exchange (124) - : +- Project (123) - : +- ShuffledHashJoin LeftSemi BuildRight (122) - : :- Exchange (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (121) - : +- Project (120) - : +- ShuffledHashJoin Inner BuildLeft (119) - : :- Exchange (105) - : : +- ShuffledHashJoin LeftSemi BuildRight (104) - : : :- Exchange (99) - : : : +- Filter (98) - : : : +- Scan parquet (97) - : : +- Exchange (103) - : : +- Project (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- Exchange (118) - : +- Filter (117) - : +- HashAggregate (116) - : +- HashAggregate (115) - : +- ShuffledHashJoin LeftSemi BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (128) - +- Project (127) - +- Filter (126) - +- Scan parquet (125) + Sort (142) + +- Exchange (141) + +- Project (140) + +- SortMergeJoin Inner (139) + :- Sort (133) + : +- Exchange (132) + : +- Project (131) + : +- SortMergeJoin LeftSemi (130) + : :- Sort (97) + : : +- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Sort (129) + : +- Exchange (128) + : +- Project (127) + : +- SortMergeJoin Inner (126) + : :- Sort (109) + : : +- Exchange (108) + : : +- SortMergeJoin LeftSemi (107) + : : :- Sort (101) + : : : +- Exchange (100) + : : : +- Filter (99) + : : : +- Scan parquet (98) + : : +- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Sort (125) + : +- Exchange (124) + : +- Filter (123) + : +- HashAggregate (122) + : +- HashAggregate (121) + : +- SortMergeJoin LeftSemi (120) + : :- Sort (114) + : : +- Exchange (113) + : : +- Project (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Sort (119) + : +- Exchange (118) + : +- Project (117) + : +- Filter (116) + : +- Scan parquet (115) + +- Sort (138) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -507,176 +517,216 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Scan parquet +(97) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(98) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(98) Filter +(99) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(99) Exchange +(100) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(101) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(102) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(103) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(104) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(105) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(106) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(107) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(109) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(110) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(107) Filter +(111) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(108) Project +(112) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(109) Exchange +(113) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(114) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(115) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(112) Project +(117) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(113) Exchange +(118) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(120) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(115) HashAggregate +(121) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(116) HashAggregate +(122) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(117) Filter +(123) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(118) Exchange +(124) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(125) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(120) Project +(127) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(121) Exchange +(128) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(129) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(130) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(123) Project +(131) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(124) Exchange +(132) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Scan parquet +(133) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(126) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(127) Project +(136) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(128) Exchange +(137) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) ShuffledHashJoin +(138) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(139) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(130) Project +(140) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(131) Exchange +(141) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(142) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(133) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index bd77a7f7f043..f92684b17b15 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (137) +- == Final Plan == VeloxColumnarToRowExec (91) +- ^ RegularHashAggregateExecTransformer (89) @@ -72,42 +72,51 @@ AdaptiveSparkPlan (128) +- ^ NoopFilter (71) +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildLeft (108) - : : :- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Exchange (107) - : : +- ShuffledHashJoin LeftAnti BuildRight (106) - : : :- ShuffledHashJoin LeftSemi BuildRight (101) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Exchange (100) - : : : +- Scan parquet (99) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Exchange (114) - : +- Project (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + TakeOrderedAndProject (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (116) + : : +- Exchange (115) + : : +- Project (114) + : : +- SortMergeJoin Inner (113) + : : :- Sort (95) + : : : +- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftAnti (110) + : : :- SortMergeJoin LeftSemi (104) + : : : :- Sort (100) + : : : : +- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (103) + : : : +- Exchange (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) (1) Scan parquet @@ -496,163 +505,199 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(95) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(100) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(100) Exchange +(102) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(103) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(102) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(103) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(104) Project +(107) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(105) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(109) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(107) Exchange +(111) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(112) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(114) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(110) Exchange +(115) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(116) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(113) Project +(119) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(114) Exchange +(120) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(121) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(116) Project +(123) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(117) Exchange +(124) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(125) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(120) Project +(128) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(129) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(132) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(124) HashAggregate +(133) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(125) Exchange +(134) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(135) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(127) TakeOrderedAndProject +(136) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(128) AdaptiveSparkPlan +(137) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 7cf55b4c0f2d..1c2790a4a999 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (52) +- == Final Plan == VeloxColumnarToRowExec (37) +- ^ SortExecTransformer (35) @@ -30,18 +30,20 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftAnti BuildRight (43) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (42) - +- Scan parquet (41) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) (1) Scan parquet @@ -210,170 +212,178 @@ Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23 Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(42) Exchange +(43) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(48) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(50) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(51) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (70) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ InputIteratorTransformer (60) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- VeloxAppendBatches (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ NoopFilter (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == - HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) -(51) Scan parquet +(53) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) NoopFilter +(54) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(53) ProjectExecTransformer +(55) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) FlushableHashAggregateExecTransformer +(56) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(56) VeloxAppendBatches +(58) VeloxAppendBatches Input [2]: [sum#X, count#X] Arguments: X -(57) ColumnarExchange +(59) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [2]: [sum#X, count#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [sum#X, count#X] -(61) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(62) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(63) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(64) Scan parquet +(66) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(65) Filter +(67) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(66) Project +(68) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(68) Exchange +(70) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(69) HashAggregate +(71) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(70) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 8b1f048c7d6e..1f9905294144 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == VeloxColumnarToRowExec (43) +- TakeOrderedAndProjectExecTransformer (42) @@ -36,25 +36,29 @@ AdaptiveSparkPlan (63) +- ^ NoopFilter (28) +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- ShuffledHashJoin Inner BuildRight (58) - :- Exchange (53) - : +- Project (52) - : +- ShuffledHashJoin Inner BuildLeft (51) - : :- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Project (63) + +- SortMergeJoin Inner (62) + :- Sort (56) + : +- Exchange (55) + : +- Project (54) + : +- SortMergeJoin Inner (53) + : :- Sort (48) + : : +- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Sort (52) + : +- Exchange (51) + : +- Filter (50) + : +- Scan parquet (49) + +- Sort (61) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -256,80 +260,96 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Scan parquet +(48) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(49) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(49) Filter +(50) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(50) Exchange +(51) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(52) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(53) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(52) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(53) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(56) Project +(59) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(57) Exchange +(60) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) ShuffledHashJoin +(61) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(62) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) Project +(63) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(60) HashAggregate +(64) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(61) HashAggregate +(65) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 1b680584826d..130bc2983040 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (56) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,21 +31,23 @@ AdaptiveSparkPlan (54) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftSemi BuildRight (47) - :- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -223,60 +225,68 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(45) Project +(46) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(46) Exchange +(47) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(50) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(51) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(54) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(55) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 67159dbb648a..10ce074fd760 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (156) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,45 +83,55 @@ AdaptiveSparkPlan (146) +- ^ NoopFilter (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- HashAggregate (143) - +- Exchange (142) - +- HashAggregate (141) - +- Project (140) - +- ShuffledHashJoin Inner BuildRight (139) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Project (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (138) - +- Project (137) - +- Filter (136) - +- Scan parquet (135) + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -567,176 +577,216 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(113) Exchange +(114) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(115) Project +(117) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(118) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(119) Exchange +(122) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(121) Project +(125) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(122) Exchange +(126) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(125) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(127) Project +(133) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(128) Exchange +(134) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(131) Exchange +(138) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(141) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(134) Exchange +(142) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(137) Project +(146) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(138) Exchange +(147) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(140) Project +(150) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(141) HashAggregate +(151) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(142) Exchange +(152) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) HashAggregate +(153) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(144) Exchange +(154) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(155) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(146) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index 71742ea423b5..651cfa840be8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (139) +AdaptiveSparkPlan (149) +- == Final Plan == VeloxColumnarToRowExec (101) +- ^ SortExecTransformer (99) @@ -79,43 +79,53 @@ AdaptiveSparkPlan (139) +- ShuffleQueryStage (79), Statistics(X) +- ReusedExchange (78) +- == Initial Plan == - Sort (138) - +- Exchange (137) - +- HashAggregate (136) - +- Exchange (135) - +- HashAggregate (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (128) - : +- Project (127) - : +- ShuffledHashJoin Inner BuildRight (126) - : :- Exchange (122) - : : +- Project (121) - : : +- ShuffledHashJoin Inner BuildRight (120) - : : :- Exchange (116) - : : : +- Project (115) - : : : +- ShuffledHashJoin Inner BuildRight (114) - : : : :- Exchange (110) - : : : : +- Project (109) - : : : : +- ShuffledHashJoin Inner BuildLeft (108) - : : : : :- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Exchange (113) - : : : +- Filter (112) - : : : +- Scan parquet (111) - : : +- Exchange (119) - : : +- Filter (118) - : : +- Scan parquet (117) - : +- Exchange (125) - : +- Filter (124) - : +- Scan parquet (123) - +- Exchange (131) - +- Filter (130) - +- Scan parquet (129) + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) (1) Scan parquet @@ -537,168 +547,208 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(107) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(108) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(111) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(110) Exchange +(112) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(112) Filter +(115) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(113) Exchange +(116) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(119) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(120) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(118) Filter +(123) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(119) Exchange +(124) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(121) Project +(127) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(122) Exchange +(128) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(124) Filter +(131) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(125) Exchange +(132) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(135) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(128) Exchange +(136) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(130) Filter +(139) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(131) Exchange +(140) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(133) Project +(143) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(134) HashAggregate +(144) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(135) Exchange +(145) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) HashAggregate +(146) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(137) Exchange +(147) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(148) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(139) AdaptiveSparkPlan +(149) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 319e6c9f1b21..e359f4c944e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (193) +AdaptiveSparkPlan (207) +- == Final Plan == VeloxColumnarToRowExec (141) +- ^ SortExecTransformer (139) @@ -110,57 +110,71 @@ AdaptiveSparkPlan (193) +- ^ NoopFilter (113) +- ^ Scan parquet (112) +- == Initial Plan == - Sort (192) - +- Exchange (191) - +- HashAggregate (190) - +- Exchange (189) - +- HashAggregate (188) - +- Project (187) - +- ShuffledHashJoin Inner BuildRight (186) - :- Exchange (181) - : +- Project (180) - : +- ShuffledHashJoin Inner BuildRight (179) - : :- Exchange (175) - : : +- Project (174) - : : +- ShuffledHashJoin Inner BuildRight (173) - : : :- Exchange (169) - : : : +- Project (168) - : : : +- ShuffledHashJoin Inner BuildRight (167) - : : : :- Exchange (163) - : : : : +- Project (162) - : : : : +- ShuffledHashJoin Inner BuildRight (161) - : : : : :- Exchange (157) - : : : : : +- Project (156) - : : : : : +- ShuffledHashJoin Inner BuildRight (155) - : : : : : :- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) - : : : : : : :- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Exchange (148) - : : : : : : +- Filter (147) - : : : : : : +- Scan parquet (146) - : : : : : +- Exchange (154) - : : : : : +- Filter (153) - : : : : : +- Scan parquet (152) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Exchange (166) - : : : +- Filter (165) - : : : +- Scan parquet (164) - : : +- Exchange (172) - : : +- Filter (171) - : : +- Scan parquet (170) - : +- Exchange (178) - : +- Filter (177) - : +- Scan parquet (176) - +- Exchange (185) - +- Project (184) - +- Filter (183) - +- Scan parquet (182) + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) (1) Scan parquet @@ -750,228 +764,284 @@ Input [2]: [p_partkey#X, p_type#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(147) Filter +(148) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(148) Exchange +(149) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(150) Project +(152) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(153) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(153) Filter +(156) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(154) Exchange +(157) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(156) Project +(160) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(157) Exchange +(161) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(159) Filter +(164) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(160) Exchange +(165) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(161) ShuffledHashJoin +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(162) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(163) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Scan parquet +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(165) Filter +(172) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(166) Exchange +(173) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(167) ShuffledHashJoin +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(168) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(169) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) Scan parquet +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(171) Filter +(180) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(172) Exchange +(181) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(173) ShuffledHashJoin +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(174) Project +(184) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(175) Exchange +(185) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Scan parquet +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(177) Filter +(188) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(178) Exchange +(189) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(179) ShuffledHashJoin +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(180) Project +(192) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(181) Exchange +(193) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(182) Scan parquet +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(183) Filter +(196) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(184) Project +(197) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(185) Exchange +(198) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(186) ShuffledHashJoin +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(187) Project +(201) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(188) HashAggregate +(202) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(189) Exchange +(203) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(190) HashAggregate +(204) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(191) Exchange +(205) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(206) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(193) AdaptiveSparkPlan +(207) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index 40dee1752399..21c91ca14180 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (155) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,44 +83,54 @@ AdaptiveSparkPlan (145) +- ^ NoopFilter (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (137) - +- Filter (136) - +- Scan parquet (135) + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -570,168 +580,208 @@ Input [2]: [p_partkey#X, p_name#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(113) Exchange +(114) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(115) Project +(117) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(118) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(119) Exchange +(122) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(121) Project +(125) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(122) Exchange +(126) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(125) Exchange +(130) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(127) Project +(133) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(128) Exchange +(134) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(131) Exchange +(138) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(133) Project +(141) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(134) Exchange +(142) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(137) Exchange +(146) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(139) Project +(149) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(140) HashAggregate +(150) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(141) Exchange +(151) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(152) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(143) Exchange +(153) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(154) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(155) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 94e1100ea37d..5e48ceb742d7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == VeloxColumnarToRowExec (67) +- TakeOrderedAndProjectExecTransformer (66) @@ -54,32 +54,38 @@ AdaptiveSparkPlan (94) +- ^ NoopFilter (45) +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- ShuffledHashJoin Inner BuildRight (88) - :- Exchange (84) - : +- Project (83) - : +- ShuffledHashJoin Inner BuildRight (82) - : :- Exchange (77) - : : +- Project (76) - : : +- ShuffledHashJoin Inner BuildRight (75) - : : :- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (81) - : +- Project (80) - : +- Filter (79) - : +- Scan parquet (78) - +- Exchange (87) - +- Filter (86) - +- Scan parquet (85) + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -374,119 +380,143 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(74) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(75) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(76) Project +(78) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(77) Exchange +(79) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(79) Filter +(82) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(80) Project +(83) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(81) Exchange +(84) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(83) Project +(87) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(84) Exchange +(88) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(87) Exchange +(92) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) ShuffledHashJoin +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(89) Project +(95) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(90) HashAggregate +(96) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(91) Exchange +(97) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(98) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) TakeOrderedAndProject +(99) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index 41d23099319f..6c9eef6b1f1f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (78) +AdaptiveSparkPlan (82) +- == Final Plan == VeloxColumnarToRowExec (56) +- ^ SortExecTransformer (54) @@ -45,27 +45,31 @@ AdaptiveSparkPlan (78) +- ^ NoopFilter (28) +- ^ Scan parquet (27) +- == Initial Plan == - Sort (77) - +- Exchange (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Project (71) - +- ShuffledHashJoin Inner BuildRight (70) - :- Exchange (65) - : +- Project (64) - : +- ShuffledHashJoin Inner BuildRight (63) - : :- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Exchange (62) - : +- Filter (61) - : +- Scan parquet (60) - +- Exchange (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) (1) Scan parquet @@ -313,365 +317,401 @@ Condition : isnotnull(ps_suppkey#X) Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) Exchange +(63) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) ShuffledHashJoin +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(64) Project +(66) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(67) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Scan parquet +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(68) Project +(71) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(69) Exchange +(72) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) ShuffledHashJoin +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) Project +(75) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(72) HashAggregate +(76) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(73) Exchange +(77) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(78) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(75) Filter +(79) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(76) Exchange +(80) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Sort +(81) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(78) AdaptiveSparkPlan +(82) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (110) - +- ^ ProjectExecTransformer (108) - +- ^ RegularHashAggregateExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ ProjectExecTransformer (105) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - :- ^ InputIteratorTransformer (99) - : +- ShuffleQueryStage (97), Statistics(X) - : +- ColumnarExchange (96) - : +- VeloxAppendBatches (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (87) - : : +- ShuffleQueryStage (85), Statistics(X) - : : +- ColumnarExchange (84) - : : +- VeloxAppendBatches (83) - : : +- ^ ProjectExecTransformer (81) - : : +- ^ NoopFilter (80) - : : +- ^ Scan parquet (79) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ReusedExchange (88) - +- ^ InputIteratorTransformer (103) - +- ShuffleQueryStage (101), Statistics(X) - +- ReusedExchange (100) + VeloxColumnarToRowExec (114) + +- ^ ProjectExecTransformer (112) + +- ^ RegularHashAggregateExecTransformer (111) + +- ^ RegularHashAggregateExecTransformer (110) + +- ^ ProjectExecTransformer (109) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) + :- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) + : :- ^ InputIteratorTransformer (91) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- ColumnarExchange (88) + : : +- VeloxAppendBatches (87) + : : +- ^ ProjectExecTransformer (85) + : : +- ^ NoopFilter (84) + : : +- ^ Scan parquet (83) + : +- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ReusedExchange (92) + +- ^ InputIteratorTransformer (107) + +- ShuffleQueryStage (105), Statistics(X) + +- ReusedExchange (104) +- == Initial Plan == - HashAggregate (127) - +- HashAggregate (126) - +- Project (125) - +- ShuffledHashJoin Inner BuildRight (124) - :- Exchange (119) - : +- Project (118) - : +- ShuffledHashJoin Inner BuildRight (117) - : :- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- Exchange (123) - +- Project (122) - +- Filter (121) - +- Scan parquet (120) - - -(79) Scan parquet + HashAggregate (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Filter (116) + : : +- Scan parquet (115) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) + + +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) NoopFilter +(84) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ProjectExecTransformer +(85) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(82) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(83) VeloxAppendBatches +(87) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(88) ReusedExchange [Reuses operator id: 15] +(92) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(94) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(95) VeloxAppendBatches +(99) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(96) ColumnarExchange +(100) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(97) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(98) InputAdapter +(102) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(100) ReusedExchange [Reuses operator id: 32] +(104) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(101) ShuffleQueryStage +(105) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(102) InputAdapter +(106) InputAdapter Input [1]: [n_nationkey#X] -(103) InputIteratorTransformer +(107) InputIteratorTransformer Input [1]: [n_nationkey#X] -(104) ShuffledHashJoinExecTransformer +(108) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(105) ProjectExecTransformer +(109) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(106) RegularHashAggregateExecTransformer +(110) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(107) RegularHashAggregateExecTransformer +(111) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(108) ProjectExecTransformer +(112) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(109) WholeStageCodegenTransformer (X) +(113) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(110) VeloxColumnarToRowExec +(114) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(111) Scan parquet +(115) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(112) Filter +(116) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(113) Exchange +(117) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) Scan parquet +(118) Sort +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(119) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(115) Filter +(120) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(116) Exchange +(121) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(118) Project +(124) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(119) Exchange +(125) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Scan parquet +(126) Sort +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(121) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(122) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(123) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) ShuffledHashJoin +(131) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(125) Project +(133) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(126) HashAggregate +(134) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(127) HashAggregate +(135) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(128) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index 9995164f4c49..8f963c49aba8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,20 +31,22 @@ AdaptiveSparkPlan (53) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin Inner BuildLeft (46) - :- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -219,61 +221,69 @@ Condition : isnotnull(o_orderkey#X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(44) Project +(45) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(45) Exchange +(46) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(47) Project +(49) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(51) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(53) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 53801198fb49..9584f92628cd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (58) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ SortExecTransformer (38) @@ -33,21 +33,23 @@ AdaptiveSparkPlan (56) +- ^ NoopFilter (10) +- ^ Scan parquet (9) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftOuter BuildRight (47) - :- Exchange (42) - : +- Scan parquet (41) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftOuter (49) + :- Sort (43) + : +- Exchange (42) + : +- Scan parquet (41) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -230,75 +232,83 @@ ReadSchema: struct Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(45) Project +(46) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(46) Exchange +(47) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(48) Project +(50) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(51) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(53) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(54) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(55) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(56) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(57) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 4080a469e711..175a0e5a97cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (39) +- == Final Plan == VeloxColumnarToRowExec (25) +- ^ ProjectExecTransformer (23) @@ -22,17 +22,19 @@ AdaptiveSparkPlan (37) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- ShuffledHashJoin Inner BuildRight (33) - :- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -163,45 +165,53 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Scan parquet +(30) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(31) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(32) Exchange +(33) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) ShuffledHashJoin +(34) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(34) Project +(36) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(36) HashAggregate +(38) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(37) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index a177fe4bcaec..130d9036b4a2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == VeloxColumnarToRowExec (30) +- AQEShuffleRead (29) @@ -27,20 +27,22 @@ AdaptiveSparkPlan (45) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -187,222 +189,230 @@ Condition : isnotnull(s_suppkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(34) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(40) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(42) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (71) +AdaptiveSparkPlan (73) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ ProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- VeloxAppendBatches (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (64) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ ProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ ProjectExecTransformer (52) + +- ^ FlushableHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ NoopFilter (49) + +- ^ Scan parquet (48) +- == Initial Plan == - HashAggregate (70) - +- HashAggregate (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (72) + +- HashAggregate (71) + +- HashAggregate (70) + +- Exchange (69) + +- HashAggregate (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) -(46) Scan parquet +(48) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(47) NoopFilter +(49) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(48) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(49) FlushableHashAggregateExecTransformer +(51) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(52) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(52) VeloxAppendBatches +(54) VeloxAppendBatches Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) ColumnarExchange +(55) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(57) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) ProjectExecTransformer +(60) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(60) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(62) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(63) Scan parquet +(65) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(64) Filter +(66) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(65) Project +(67) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(67) Exchange +(69) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(70) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(69) HashAggregate +(71) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(70) HashAggregate +(72) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(71) AdaptiveSparkPlan +(73) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 89a647ffce45..32a24beb94b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (71) +- == Final Plan == VeloxColumnarToRowExec (47) +- ^ SortExecTransformer (45) @@ -38,27 +38,29 @@ AdaptiveSparkPlan (69) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- ShuffledHashJoin Inner BuildRight (59) - :- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (58) - +- Filter (57) - +- Scan parquet (56) + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -298,75 +300,83 @@ Join condition: None Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(58) Exchange +(59) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) ShuffledHashJoin +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(60) Project +(62) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(61) HashAggregate +(63) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(65) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(67) Exchange +(69) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(70) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index 42fc32b0bce1..59baa2d7a08c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (63) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ ProjectExecTransformer (38) @@ -35,25 +35,28 @@ AdaptiveSparkPlan (60) +- ^ NoopFilter (22) +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (59) - +- HashAggregate (58) - +- Project (57) - +- ShuffledHashJoin Inner BuildRight (56) - :- Project (49) - : +- ShuffledHashJoin Inner BuildRight (48) - : :- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Filter (51) - +- Scan parquet (50) + HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (44) + : : +- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -249,92 +252,104 @@ Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(44) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(45) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(46) Project +(47) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(47) Exchange +(48) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(49) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(49) Project +(51) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(50) Scan parquet +(52) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(51) Filter +(53) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(52) HashAggregate +(54) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(53) Exchange +(55) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) Filter +(57) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(56) ShuffledHashJoin +(58) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(57) Project +(60) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(58) HashAggregate +(61) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(59) HashAggregate +(62) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(60) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index ca913b2d4c84..c78e265e54d9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (110) +- == Final Plan == VeloxColumnarToRowExec (70) +- TakeOrderedAndProjectExecTransformer (69) @@ -58,38 +58,45 @@ AdaptiveSparkPlan (103) +- ShuffleQueryStage (57), Statistics(X) +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (109) + +- HashAggregate (108) + +- HashAggregate (107) + +- Project (106) + +- SortMergeJoin Inner (105) + :- Sort (92) + : +- Exchange (91) + : +- Project (90) + : +- SortMergeJoin Inner (89) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (88) + : +- Exchange (87) + : +- SortMergeJoin LeftSemi (86) + : :- Sort (78) + : : +- Exchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Sort (85) + : +- Project (84) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- Scan parquet (79) + +- SortMergeJoin LeftSemi (104) + :- Sort (96) + : +- Exchange (95) + : +- Filter (94) + : +- Scan parquet (93) + +- Sort (103) + +- Project (102) + +- Filter (101) + +- HashAggregate (100) + +- Exchange (99) + +- HashAggregate (98) + +- Scan parquet (97) (1) Scan parquet @@ -405,158 +412,186 @@ Condition : isnotnull(c_custkey#X) Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(77) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(80) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(81) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(82) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(83) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(84) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(85) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(84) Exchange +(87) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(88) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(89) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(86) Project +(90) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(91) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(92) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(93) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(98) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(99) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(100) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(101) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(102) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(103) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(98) ShuffledHashJoin +(105) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(99) Project +(106) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(107) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(108) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(109) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 91187ac8d5a7..a9c629524fb7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (36) +AdaptiveSparkPlan (38) +- == Final Plan == VeloxColumnarToRowExec (24) +- ^ RegularHashAggregateExecTransformer (22) @@ -21,17 +21,19 @@ AdaptiveSparkPlan (36) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (35) - +- HashAggregate (34) - +- Project (33) - +- ShuffledHashJoin Inner BuildRight (32) - :- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -158,45 +160,53 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(29) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(30) Filter +(31) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(31) Exchange +(32) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) ShuffledHashJoin +(33) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(33) Project +(35) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(34) HashAggregate +(36) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(36) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 01ab88ee0b2b..8e929ff7b296 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (143) +- == Final Plan == VeloxColumnarToRowExec (93) +- AQEShuffleRead (92) @@ -75,45 +75,55 @@ AdaptiveSparkPlan (133) +- ^ NoopFilter (78) +- ^ Scan parquet (77) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- Project (130) - +- ShuffledHashJoin Inner BuildRight (129) - :- Exchange (124) - : +- Project (123) - : +- ShuffledHashJoin LeftSemi BuildRight (122) - : :- Exchange (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (121) - : +- Project (120) - : +- ShuffledHashJoin Inner BuildLeft (119) - : :- Exchange (105) - : : +- ShuffledHashJoin LeftSemi BuildRight (104) - : : :- Exchange (99) - : : : +- Filter (98) - : : : +- Scan parquet (97) - : : +- Exchange (103) - : : +- Project (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- Exchange (118) - : +- Filter (117) - : +- HashAggregate (116) - : +- HashAggregate (115) - : +- ShuffledHashJoin LeftSemi BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (128) - +- Project (127) - +- Filter (126) - +- Scan parquet (125) + Sort (142) + +- Exchange (141) + +- Project (140) + +- SortMergeJoin Inner (139) + :- Sort (133) + : +- Exchange (132) + : +- Project (131) + : +- SortMergeJoin LeftSemi (130) + : :- Sort (97) + : : +- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Sort (129) + : +- Exchange (128) + : +- Project (127) + : +- SortMergeJoin Inner (126) + : :- Sort (109) + : : +- Exchange (108) + : : +- SortMergeJoin LeftSemi (107) + : : :- Sort (101) + : : : +- Exchange (100) + : : : +- Filter (99) + : : : +- Scan parquet (98) + : : +- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Sort (125) + : +- Exchange (124) + : +- Filter (123) + : +- HashAggregate (122) + : +- HashAggregate (121) + : +- SortMergeJoin LeftSemi (120) + : :- Sort (114) + : : +- Exchange (113) + : : +- Project (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Sort (119) + : +- Exchange (118) + : +- Project (117) + : +- Filter (116) + : +- Scan parquet (115) + +- Sort (138) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -512,181 +522,221 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Scan parquet +(97) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(98) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(98) Filter +(99) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(99) Exchange +(100) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(101) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(102) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(103) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(104) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(105) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(106) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(107) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(109) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(110) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(107) Filter +(111) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(108) Project +(112) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(109) Exchange +(113) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(114) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(115) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(112) Project +(117) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(113) Exchange +(118) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(120) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(115) HashAggregate +(121) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(116) HashAggregate +(122) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(117) Filter +(123) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(118) Exchange +(124) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(125) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(120) Project +(127) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(121) Exchange +(128) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(129) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(130) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(123) Project +(131) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(124) Exchange +(132) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Scan parquet +(133) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(126) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(127) Project +(136) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(128) Exchange +(137) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) ShuffledHashJoin +(138) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(139) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(130) Project +(140) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(131) Exchange +(141) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(142) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(133) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 317740080d7b..279f4f096692 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (137) +- == Final Plan == VeloxColumnarToRowExec (91) +- ^ RegularHashAggregateExecTransformer (89) @@ -72,42 +72,51 @@ AdaptiveSparkPlan (128) +- ^ NoopFilter (71) +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildLeft (108) - : : :- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Exchange (107) - : : +- ShuffledHashJoin LeftAnti BuildRight (106) - : : :- ShuffledHashJoin LeftSemi BuildRight (101) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Exchange (100) - : : : +- Scan parquet (99) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Exchange (114) - : +- Project (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + TakeOrderedAndProject (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (116) + : : +- Exchange (115) + : : +- Project (114) + : : +- SortMergeJoin Inner (113) + : : :- Sort (95) + : : : +- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftAnti (110) + : : :- SortMergeJoin LeftSemi (104) + : : : :- Sort (100) + : : : : +- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (103) + : : : +- Exchange (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) (1) Scan parquet @@ -501,168 +510,204 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(95) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(100) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(100) Exchange +(102) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(103) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(102) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(103) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(104) Project +(107) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(105) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(109) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(107) Exchange +(111) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(112) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(109) Project +(114) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(110) Exchange +(115) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(116) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(113) Project +(119) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(114) Exchange +(120) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(121) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(116) Project +(123) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(117) Exchange +(124) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(125) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(120) Project +(128) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(129) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(123) Project +(132) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(124) HashAggregate +(133) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(125) Exchange +(134) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(135) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(127) TakeOrderedAndProject +(136) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(128) AdaptiveSparkPlan +(137) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 0d779c9160cf..2b93055014bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (52) +- == Final Plan == VeloxColumnarToRowExec (37) +- ^ SortExecTransformer (35) @@ -30,18 +30,20 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftAnti BuildRight (43) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (42) - +- Scan parquet (41) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) (1) Scan parquet @@ -211,171 +213,179 @@ Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23 Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(42) Exchange +(43) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(44) Project +(46) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(48) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(50) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(51) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (70) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ InputIteratorTransformer (60) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- VeloxAppendBatches (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ NoopFilter (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ NoopFilter (54) + +- ^ Scan parquet (53) +- == Initial Plan == - HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) -(51) Scan parquet +(53) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) NoopFilter +(54) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(53) ProjectExecTransformer +(55) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) FlushableHashAggregateExecTransformer +(56) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(56) VeloxAppendBatches +(58) VeloxAppendBatches Input [2]: [sum#X, count#X] Arguments: X -(57) ColumnarExchange +(59) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [2]: [sum#X, count#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [sum#X, count#X] -(61) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(62) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(63) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(64) Scan parquet +(66) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(65) Filter +(67) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(66) Project +(68) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(68) Exchange +(70) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(69) HashAggregate +(71) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(70) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index 8c671a61c9f7..aa679861da7c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == VeloxColumnarToRowExec (43) +- TakeOrderedAndProjectExecTransformer (42) @@ -36,25 +36,29 @@ AdaptiveSparkPlan (63) +- ^ NoopFilter (28) +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- ShuffledHashJoin Inner BuildRight (58) - :- Exchange (53) - : +- Project (52) - : +- ShuffledHashJoin Inner BuildLeft (51) - : :- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Project (63) + +- SortMergeJoin Inner (62) + :- Sort (56) + : +- Exchange (55) + : +- Project (54) + : +- SortMergeJoin Inner (53) + : :- Sort (48) + : : +- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Sort (52) + : +- Exchange (51) + : +- Filter (50) + : +- Scan parquet (49) + +- Sort (61) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -258,82 +262,98 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Scan parquet +(48) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(49) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(49) Filter +(50) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(50) Exchange +(51) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(52) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(53) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(52) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(53) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(56) Project +(59) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(57) Exchange +(60) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) ShuffledHashJoin +(61) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(62) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) Project +(63) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(60) HashAggregate +(64) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(61) HashAggregate +(65) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 3d145f0c3bb8..02c494288f95 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (56) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,21 +31,23 @@ AdaptiveSparkPlan (54) +- ^ NoopFilter (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftSemi BuildRight (47) - :- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -224,61 +226,69 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(45) Project +(46) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(46) Exchange +(47) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(48) Project +(50) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(51) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(54) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(55) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 08e655f5aa81..67150984ab61 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (156) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,45 +83,55 @@ AdaptiveSparkPlan (146) +- ^ NoopFilter (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- HashAggregate (143) - +- Exchange (142) - +- HashAggregate (141) - +- Project (140) - +- ShuffledHashJoin Inner BuildRight (139) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Project (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (138) - +- Project (137) - +- Filter (136) - +- Scan parquet (135) + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -572,181 +582,221 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(113) Exchange +(114) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(115) Project +(117) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(118) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(119) Exchange +(122) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(121) Project +(125) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(122) Exchange +(126) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(125) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(127) Project +(133) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(128) Exchange +(134) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(131) Exchange +(138) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(133) Project +(141) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(134) Exchange +(142) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(137) Project +(146) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(138) Exchange +(147) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(140) Project +(150) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(141) HashAggregate +(151) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(142) Exchange +(152) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) HashAggregate +(153) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(144) Exchange +(154) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(155) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(146) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 71ade94be21d..65dfab993c3c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (139) +AdaptiveSparkPlan (149) +- == Final Plan == VeloxColumnarToRowExec (101) +- ^ SortExecTransformer (99) @@ -79,43 +79,53 @@ AdaptiveSparkPlan (139) +- ShuffleQueryStage (79), Statistics(X) +- ReusedExchange (78) +- == Initial Plan == - Sort (138) - +- Exchange (137) - +- HashAggregate (136) - +- Exchange (135) - +- HashAggregate (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (128) - : +- Project (127) - : +- ShuffledHashJoin Inner BuildRight (126) - : :- Exchange (122) - : : +- Project (121) - : : +- ShuffledHashJoin Inner BuildRight (120) - : : :- Exchange (116) - : : : +- Project (115) - : : : +- ShuffledHashJoin Inner BuildRight (114) - : : : :- Exchange (110) - : : : : +- Project (109) - : : : : +- ShuffledHashJoin Inner BuildLeft (108) - : : : : :- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Exchange (113) - : : : +- Filter (112) - : : : +- Scan parquet (111) - : : +- Exchange (119) - : : +- Filter (118) - : : +- Scan parquet (117) - : +- Exchange (125) - : +- Filter (124) - : +- Scan parquet (123) - +- Exchange (131) - +- Filter (130) - +- Scan parquet (129) + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) (1) Scan parquet @@ -542,173 +552,213 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(107) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(108) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(109) Project +(111) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(110) Exchange +(112) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(112) Filter +(115) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(113) Exchange +(116) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(115) Project +(119) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(120) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(118) Filter +(123) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(119) Exchange +(124) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(121) Project +(127) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(122) Exchange +(128) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(124) Filter +(131) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(125) Exchange +(132) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(135) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(128) Exchange +(136) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(130) Filter +(139) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(131) Exchange +(140) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(133) Project +(143) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(134) HashAggregate +(144) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(135) Exchange +(145) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) HashAggregate +(146) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(137) Exchange +(147) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(148) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(139) AdaptiveSparkPlan +(149) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index ddeab25c4569..61f6287c2429 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (193) +AdaptiveSparkPlan (207) +- == Final Plan == VeloxColumnarToRowExec (141) +- ^ SortExecTransformer (139) @@ -110,57 +110,71 @@ AdaptiveSparkPlan (193) +- ^ NoopFilter (113) +- ^ Scan parquet (112) +- == Initial Plan == - Sort (192) - +- Exchange (191) - +- HashAggregate (190) - +- Exchange (189) - +- HashAggregate (188) - +- Project (187) - +- ShuffledHashJoin Inner BuildRight (186) - :- Exchange (181) - : +- Project (180) - : +- ShuffledHashJoin Inner BuildRight (179) - : :- Exchange (175) - : : +- Project (174) - : : +- ShuffledHashJoin Inner BuildRight (173) - : : :- Exchange (169) - : : : +- Project (168) - : : : +- ShuffledHashJoin Inner BuildRight (167) - : : : :- Exchange (163) - : : : : +- Project (162) - : : : : +- ShuffledHashJoin Inner BuildRight (161) - : : : : :- Exchange (157) - : : : : : +- Project (156) - : : : : : +- ShuffledHashJoin Inner BuildRight (155) - : : : : : :- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) - : : : : : : :- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Exchange (148) - : : : : : : +- Filter (147) - : : : : : : +- Scan parquet (146) - : : : : : +- Exchange (154) - : : : : : +- Filter (153) - : : : : : +- Scan parquet (152) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Exchange (166) - : : : +- Filter (165) - : : : +- Scan parquet (164) - : : +- Exchange (172) - : : +- Filter (171) - : : +- Scan parquet (170) - : +- Exchange (178) - : +- Filter (177) - : +- Scan parquet (176) - +- Exchange (185) - +- Project (184) - +- Filter (183) - +- Scan parquet (182) + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) (1) Scan parquet @@ -757,235 +771,291 @@ Input [2]: [p_partkey#X, p_type#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(147) Filter +(148) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(148) Exchange +(149) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(150) Project +(152) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(153) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(153) Filter +(156) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(154) Exchange +(157) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(156) Project +(160) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(157) Exchange +(161) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(159) Filter +(164) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(160) Exchange +(165) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(161) ShuffledHashJoin +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(162) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(163) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Scan parquet +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(165) Filter +(172) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(166) Exchange +(173) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(167) ShuffledHashJoin +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(168) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(169) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) Scan parquet +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(171) Filter +(180) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(172) Exchange +(181) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(173) ShuffledHashJoin +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(174) Project +(184) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(175) Exchange +(185) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Scan parquet +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(177) Filter +(188) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(178) Exchange +(189) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(179) ShuffledHashJoin +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(180) Project +(192) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(181) Exchange +(193) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(182) Scan parquet +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(183) Filter +(196) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(184) Project +(197) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(185) Exchange +(198) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(186) ShuffledHashJoin +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(187) Project +(201) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(188) HashAggregate +(202) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(189) Exchange +(203) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(190) HashAggregate +(204) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(191) Exchange +(205) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(206) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(193) AdaptiveSparkPlan +(207) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 634e3516a710..4b983de23fde 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (155) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,44 +83,54 @@ AdaptiveSparkPlan (145) +- ^ NoopFilter (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (137) - +- Filter (136) - +- Scan parquet (135) + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -575,173 +585,213 @@ Input [2]: [p_partkey#X, p_name#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(113) Exchange +(114) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(115) Project +(117) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(118) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(119) Exchange +(122) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(121) Project +(125) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(122) Exchange +(126) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(125) Exchange +(130) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(127) Project +(133) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(128) Exchange +(134) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(131) Exchange +(138) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(133) Project +(141) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(134) Exchange +(142) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(137) Exchange +(146) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(139) Project +(149) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(140) HashAggregate +(150) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(141) Exchange +(151) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(152) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(143) Exchange +(153) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(154) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(155) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index c5fcd91867cb..993884df3f3a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == VeloxColumnarToRowExec (67) +- TakeOrderedAndProjectExecTransformer (66) @@ -54,32 +54,38 @@ AdaptiveSparkPlan (94) +- ^ FilterExecTransformer (45) +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- ShuffledHashJoin Inner BuildRight (88) - :- Exchange (84) - : +- Project (83) - : +- ShuffledHashJoin Inner BuildRight (82) - : :- Exchange (77) - : : +- Project (76) - : : +- ShuffledHashJoin Inner BuildRight (75) - : : :- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (81) - : +- Project (80) - : +- Filter (79) - : +- Scan parquet (78) - +- Exchange (87) - +- Filter (86) - +- Scan parquet (85) + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -371,116 +377,140 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(74) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(75) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(76) Project +(78) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(77) Exchange +(79) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(79) Filter +(82) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(80) Project +(83) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(81) Exchange +(84) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Project +(87) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(84) Exchange +(88) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(87) Exchange +(92) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) ShuffledHashJoin +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(95) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(90) HashAggregate +(96) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(91) Exchange +(97) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(98) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) TakeOrderedAndProject +(99) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index 59de06707aad..8142375d9ead 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (78) +AdaptiveSparkPlan (82) +- == Final Plan == VeloxColumnarToRowExec (56) +- ^ SortExecTransformer (54) @@ -45,27 +45,31 @@ AdaptiveSparkPlan (78) +- ^ FilterExecTransformer (28) +- ^ Scan parquet (27) +- == Initial Plan == - Sort (77) - +- Exchange (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Project (71) - +- ShuffledHashJoin Inner BuildRight (70) - :- Exchange (65) - : +- Project (64) - : +- ShuffledHashJoin Inner BuildRight (63) - : :- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Exchange (62) - : +- Filter (61) - : +- Scan parquet (60) - +- Exchange (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) (1) Scan parquet @@ -311,92 +315,108 @@ Condition : isnotnull(ps_suppkey#X) Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) Exchange +(63) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) ShuffledHashJoin +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(64) Project +(66) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(67) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Scan parquet +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(68) Project +(71) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(69) Exchange +(72) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) ShuffledHashJoin +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) Project +(75) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(72) HashAggregate +(76) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(73) Exchange +(77) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(78) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(75) Filter +(79) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(76) Exchange +(80) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Sort +(81) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(78) AdaptiveSparkPlan +(82) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index a8ac5d0d2c1b..802a79759235 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,20 +31,22 @@ AdaptiveSparkPlan (53) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin Inner BuildLeft (46) - :- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -218,60 +220,68 @@ Condition : isnotnull(o_orderkey#X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(44) Project +(45) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(45) Exchange +(46) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(49) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(51) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(53) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index d65867ecf822..b9bf0f1fad60 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (58) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ SortExecTransformer (38) @@ -33,21 +33,23 @@ AdaptiveSparkPlan (56) +- ^ FilterExecTransformer (10) +- ^ Scan parquet (9) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftOuter BuildRight (47) - :- Exchange (42) - : +- Scan parquet (41) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftOuter (49) + :- Sort (43) + : +- Exchange (42) + : +- Scan parquet (41) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -229,74 +231,82 @@ ReadSchema: struct Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(45) Project +(46) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(46) Exchange +(47) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(50) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(51) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(53) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(54) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(55) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(56) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(57) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt index 2bc0be8fcb67..425c55f5a4ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (39) +- == Final Plan == VeloxColumnarToRowExec (25) +- ^ ProjectExecTransformer (23) @@ -22,17 +22,19 @@ AdaptiveSparkPlan (37) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- ShuffledHashJoin Inner BuildRight (33) - :- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -162,44 +164,52 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Scan parquet +(30) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(31) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(32) Exchange +(33) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) ShuffledHashJoin +(34) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) Project +(36) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(36) HashAggregate +(38) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(37) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index 0d21930825c7..f003eed994d0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (48) +AdaptiveSparkPlan (50) +- == Final Plan == VeloxColumnarToRowExec (33) +- ^ SortExecTransformer (31) @@ -28,20 +28,22 @@ AdaptiveSparkPlan (48) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (47) - +- Exchange (46) - +- Project (45) - +- ShuffledHashJoin Inner BuildLeft (44) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Filter (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (49) + +- Exchange (48) + +- Project (47) + +- SortMergeJoin Inner (46) + :- Sort (37) + : +- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Sort (45) + +- Filter (44) + +- HashAggregate (43) + +- Exchange (42) + +- HashAggregate (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -197,60 +199,68 @@ Condition : isnotnull(s_suppkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(37) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(38) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(38) Filter +(39) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(39) Project +(40) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(40) HashAggregate +(41) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(41) Exchange +(42) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(43) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(43) Filter +(44) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(44) ShuffledHashJoin +(45) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(46) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(45) Project +(47) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(46) Exchange +(48) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) Sort +(49) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(48) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index cd3e53ad7bd6..c9374b01ff02 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (71) +- == Final Plan == VeloxColumnarToRowExec (47) +- ^ SortExecTransformer (45) @@ -38,27 +38,29 @@ AdaptiveSparkPlan (69) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- ShuffledHashJoin Inner BuildRight (59) - :- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (58) - +- Filter (57) - +- Scan parquet (56) + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -296,74 +298,82 @@ Join condition: None Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(58) Exchange +(59) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) ShuffledHashJoin +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(60) Project +(62) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(61) HashAggregate +(63) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(65) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(67) Exchange +(69) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(70) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index fc17c87d7df0..69f50fa16ab0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (63) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ ProjectExecTransformer (38) @@ -35,25 +35,28 @@ AdaptiveSparkPlan (60) +- ^ FilterExecTransformer (22) +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (59) - +- HashAggregate (58) - +- Project (57) - +- ShuffledHashJoin Inner BuildRight (56) - :- Project (49) - : +- ShuffledHashJoin Inner BuildRight (48) - : :- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Filter (51) - +- Scan parquet (50) + HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (44) + : : +- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -247,90 +250,102 @@ Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(44) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(45) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(46) Project +(47) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(47) Exchange +(48) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(49) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(49) Project +(51) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(50) Scan parquet +(52) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(51) Filter +(53) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(52) HashAggregate +(54) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(53) Exchange +(55) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) Filter +(57) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(56) ShuffledHashJoin +(58) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(57) Project +(60) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(58) HashAggregate +(61) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(59) HashAggregate +(62) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(60) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index fc65f4b52897..96fa1cd82606 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (110) +- == Final Plan == VeloxColumnarToRowExec (70) +- TakeOrderedAndProjectExecTransformer (69) @@ -58,38 +58,45 @@ AdaptiveSparkPlan (103) +- ShuffleQueryStage (57) +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (109) + +- HashAggregate (108) + +- HashAggregate (107) + +- Project (106) + +- SortMergeJoin Inner (105) + :- Sort (92) + : +- Exchange (91) + : +- Project (90) + : +- SortMergeJoin Inner (89) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (88) + : +- Exchange (87) + : +- SortMergeJoin LeftSemi (86) + : :- Sort (78) + : : +- Exchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Sort (85) + : +- Project (84) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- Scan parquet (79) + +- SortMergeJoin LeftSemi (104) + :- Sort (96) + : +- Exchange (95) + : +- Filter (94) + : +- Scan parquet (93) + +- Sort (103) + +- Project (102) + +- Filter (101) + +- HashAggregate (100) + +- Exchange (99) + +- HashAggregate (98) + +- Scan parquet (97) (1) Scan parquet @@ -401,154 +408,182 @@ Condition : isnotnull(c_custkey#X) Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(77) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(80) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(81) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(82) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(83) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(84) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(85) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(84) Exchange +(87) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(88) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(89) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(86) Project +(90) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(91) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(92) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(93) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(98) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(99) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(100) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(101) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(102) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(103) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) ShuffledHashJoin +(105) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(99) Project +(106) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(107) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(108) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(109) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt index d3d74c5ba792..ae0feb5dfd56 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (36) +AdaptiveSparkPlan (38) +- == Final Plan == VeloxColumnarToRowExec (24) +- ^ RegularHashAggregateExecTransformer (22) @@ -21,17 +21,19 @@ AdaptiveSparkPlan (36) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (35) - +- HashAggregate (34) - +- Project (33) - +- ShuffledHashJoin Inner BuildRight (32) - :- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -157,44 +159,52 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(29) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(30) Filter +(31) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(31) Exchange +(32) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) ShuffledHashJoin +(33) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(33) Project +(35) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(34) HashAggregate +(36) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(36) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt index df1ae98f903e..bab785551636 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (136) +AdaptiveSparkPlan (146) +- == Final Plan == VeloxColumnarToRowExec (96) +- ^ SortExecTransformer (94) @@ -76,45 +76,55 @@ AdaptiveSparkPlan (136) +- ^ FilterExecTransformer (78) +- ^ Scan parquet (77) +- == Initial Plan == - Sort (135) - +- Exchange (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (127) - : +- Project (126) - : +- ShuffledHashJoin LeftSemi BuildRight (125) - : :- Exchange (99) - : : +- Filter (98) - : : +- Scan parquet (97) - : +- Exchange (124) - : +- Project (123) - : +- ShuffledHashJoin Inner BuildLeft (122) - : :- Exchange (108) - : : +- ShuffledHashJoin LeftSemi BuildRight (107) - : : :- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (106) - : : +- Project (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (121) - : +- Filter (120) - : +- HashAggregate (119) - : +- HashAggregate (118) - : +- ShuffledHashJoin LeftSemi BuildRight (117) - : :- Exchange (112) - : : +- Project (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- Exchange (116) - : +- Project (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Exchange (131) - +- Project (130) - +- Filter (129) - +- Scan parquet (128) + Sort (145) + +- Exchange (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (136) + : +- Exchange (135) + : +- Project (134) + : +- SortMergeJoin LeftSemi (133) + : :- Sort (100) + : : +- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Sort (132) + : +- Exchange (131) + : +- Project (130) + : +- SortMergeJoin Inner (129) + : :- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftSemi (110) + : : :- Sort (104) + : : : +- Exchange (103) + : : : +- Filter (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (128) + : +- Exchange (127) + : +- Filter (126) + : +- HashAggregate (125) + : +- HashAggregate (124) + : +- SortMergeJoin LeftSemi (123) + : :- Sort (117) + : : +- Exchange (116) + : : +- Project (115) + : : +- Filter (114) + : : +- Scan parquet (113) + : +- Sort (122) + : +- Exchange (121) + : +- Project (120) + : +- Filter (119) + : +- Scan parquet (118) + +- Sort (141) + +- Exchange (140) + +- Project (139) + +- Filter (138) + +- Scan parquet (137) (1) Scan parquet @@ -518,176 +528,216 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(100) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(101) Filter +(102) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(102) Exchange +(103) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(104) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(105) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(106) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(107) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(108) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(109) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) Exchange +(111) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(112) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(113) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(110) Filter +(114) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(111) Project +(115) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(112) Exchange +(116) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Scan parquet +(117) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(118) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(114) Filter +(119) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(115) Project +(120) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(116) Exchange +(121) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(118) HashAggregate +(124) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(119) HashAggregate +(125) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(120) Filter +(126) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(121) Exchange +(127) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(128) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(129) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(123) Project +(130) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(124) Exchange +(131) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) ShuffledHashJoin +(132) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(133) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(126) Project +(134) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(127) Exchange +(135) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Scan parquet +(136) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(137) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(129) Filter +(138) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(130) Project +(139) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(131) Exchange +(140) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(143) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(134) Exchange +(144) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Sort +(145) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(136) AdaptiveSparkPlan +(146) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index c0f3602f9fe6..ef4e87bb1de4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (138) +- == Final Plan == VeloxColumnarToRowExec (92) +- TakeOrderedAndProjectExecTransformer (91) @@ -73,42 +73,51 @@ AdaptiveSparkPlan (129) +- ^ FilterExecTransformer (71) +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (128) - +- HashAggregate (127) - +- Exchange (126) - +- HashAggregate (125) - +- Project (124) - +- ShuffledHashJoin Inner BuildRight (123) - :- Exchange (118) - : +- Project (117) - : +- ShuffledHashJoin Inner BuildRight (116) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildLeft (109) - : : :- Exchange (95) - : : : +- Filter (94) - : : : +- Scan parquet (93) - : : +- Exchange (108) - : : +- ShuffledHashJoin LeftAnti BuildRight (107) - : : :- ShuffledHashJoin LeftSemi BuildRight (102) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- Filter (97) - : : : : +- Scan parquet (96) - : : : +- Exchange (101) - : : : +- Scan parquet (100) - : : +- Exchange (106) - : : +- Project (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (115) - : +- Project (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (122) - +- Project (121) - +- Filter (120) - +- Scan parquet (119) + TakeOrderedAndProject (137) + +- HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (117) + : : +- Exchange (116) + : : +- Project (115) + : : +- SortMergeJoin Inner (114) + : : :- Sort (96) + : : : +- Exchange (95) + : : : +- Filter (94) + : : : +- Scan parquet (93) + : : +- Sort (113) + : : +- Exchange (112) + : : +- SortMergeJoin LeftAnti (111) + : : :- SortMergeJoin LeftSemi (105) + : : : :- Sort (101) + : : : : +- Exchange (100) + : : : : +- Project (99) + : : : : +- Filter (98) + : : : : +- Scan parquet (97) + : : : +- Sort (104) + : : : +- Exchange (103) + : : : +- Scan parquet (102) + : : +- Sort (110) + : : +- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Sort (122) + : +- Exchange (121) + : +- Project (120) + : +- Filter (119) + : +- Scan parquet (118) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) (1) Scan parquet @@ -501,163 +510,199 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(96) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(97) Filter +(98) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(98) Project +(99) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(99) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(101) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(102) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(101) Exchange +(103) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(104) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(105) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(103) Scan parquet +(106) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(104) Filter +(107) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(105) Project +(108) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(106) Exchange +(109) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(110) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(111) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(108) Exchange +(112) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(113) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(114) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(110) Project +(115) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(111) Exchange +(116) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(117) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(118) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(113) Filter +(119) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(114) Project +(120) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(115) Exchange +(121) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(122) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(117) Project +(124) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(118) Exchange +(125) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Scan parquet +(126) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(120) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(121) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(122) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) ShuffledHashJoin +(131) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(124) Project +(133) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(125) HashAggregate +(134) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(126) Exchange +(135) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) HashAggregate +(136) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(128) TakeOrderedAndProject +(137) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(129) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index 5ab0811e658f..fcf712a9d5fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (52) +- == Final Plan == VeloxColumnarToRowExec (37) +- ^ SortExecTransformer (35) @@ -30,18 +30,20 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftAnti BuildRight (43) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (42) - +- Scan parquet (41) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) (1) Scan parquet @@ -210,51 +212,59 @@ Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23 Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(42) Exchange +(43) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(48) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(50) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(51) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index c51701bd0840..607d6444f432 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == VeloxColumnarToRowExec (43) +- TakeOrderedAndProjectExecTransformer (42) @@ -36,25 +36,29 @@ AdaptiveSparkPlan (63) +- ^ FilterExecTransformer (28) +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- ShuffledHashJoin Inner BuildRight (58) - :- Exchange (53) - : +- Project (52) - : +- ShuffledHashJoin Inner BuildLeft (51) - : :- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Project (63) + +- SortMergeJoin Inner (62) + :- Sort (56) + : +- Exchange (55) + : +- Project (54) + : +- SortMergeJoin Inner (53) + : :- Sort (48) + : : +- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Sort (52) + : +- Exchange (51) + : +- Filter (50) + : +- Scan parquet (49) + +- Sort (61) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -256,80 +260,96 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Scan parquet +(48) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(49) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(49) Filter +(50) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(50) Exchange +(51) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(52) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(53) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(52) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(53) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(56) Project +(59) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(57) Exchange +(60) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) ShuffledHashJoin +(61) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(62) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) Project +(63) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(60) HashAggregate +(64) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(61) HashAggregate +(65) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index 1b95ae3dbf39..cc6b8f351600 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (56) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,21 +31,23 @@ AdaptiveSparkPlan (54) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftSemi BuildRight (47) - :- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -223,60 +225,68 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(45) Project +(46) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(46) Exchange +(47) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(50) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(51) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(54) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(55) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index c31fbccc1e59..a1f95887aae3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (156) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,45 +83,55 @@ AdaptiveSparkPlan (146) +- ^ FilterExecTransformer (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- HashAggregate (143) - +- Exchange (142) - +- HashAggregate (141) - +- Project (140) - +- ShuffledHashJoin Inner BuildRight (139) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Project (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (138) - +- Project (137) - +- Filter (136) - +- Scan parquet (135) + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -567,176 +577,216 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(113) Exchange +(114) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(115) Project +(117) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(118) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(119) Exchange +(122) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(121) Project +(125) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(122) Exchange +(126) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(125) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(127) Project +(133) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(128) Exchange +(134) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(131) Exchange +(138) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(141) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(134) Exchange +(142) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(137) Project +(146) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(138) Exchange +(147) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(140) Project +(150) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(141) HashAggregate +(151) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(142) Exchange +(152) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) HashAggregate +(153) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(144) Exchange +(154) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(155) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(146) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index 06b84fdca2c7..64d51413a084 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (139) +AdaptiveSparkPlan (149) +- == Final Plan == VeloxColumnarToRowExec (101) +- ^ SortExecTransformer (99) @@ -79,43 +79,53 @@ AdaptiveSparkPlan (139) +- ShuffleQueryStage (79) +- ReusedExchange (78) +- == Initial Plan == - Sort (138) - +- Exchange (137) - +- HashAggregate (136) - +- Exchange (135) - +- HashAggregate (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (128) - : +- Project (127) - : +- ShuffledHashJoin Inner BuildRight (126) - : :- Exchange (122) - : : +- Project (121) - : : +- ShuffledHashJoin Inner BuildRight (120) - : : :- Exchange (116) - : : : +- Project (115) - : : : +- ShuffledHashJoin Inner BuildRight (114) - : : : :- Exchange (110) - : : : : +- Project (109) - : : : : +- ShuffledHashJoin Inner BuildLeft (108) - : : : : :- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Exchange (113) - : : : +- Filter (112) - : : : +- Scan parquet (111) - : : +- Exchange (119) - : : +- Filter (118) - : : +- Scan parquet (117) - : +- Exchange (125) - : +- Filter (124) - : +- Scan parquet (123) - +- Exchange (131) - +- Filter (130) - +- Scan parquet (129) + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) (1) Scan parquet @@ -537,168 +547,208 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(107) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(108) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(111) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(110) Exchange +(112) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(112) Filter +(115) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(113) Exchange +(116) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(119) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(120) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(118) Filter +(123) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(119) Exchange +(124) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(121) Project +(127) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(122) Exchange +(128) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(124) Filter +(131) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(125) Exchange +(132) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(135) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(128) Exchange +(136) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(130) Filter +(139) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(131) Exchange +(140) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(133) Project +(143) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(134) HashAggregate +(144) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(135) Exchange +(145) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) HashAggregate +(146) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(137) Exchange +(147) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(148) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(139) AdaptiveSparkPlan +(149) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index e9fdc420f128..8934d1a2e7a3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (193) +AdaptiveSparkPlan (207) +- == Final Plan == VeloxColumnarToRowExec (141) +- ^ SortExecTransformer (139) @@ -110,57 +110,71 @@ AdaptiveSparkPlan (193) +- ^ FilterExecTransformer (113) +- ^ Scan parquet (112) +- == Initial Plan == - Sort (192) - +- Exchange (191) - +- HashAggregate (190) - +- Exchange (189) - +- HashAggregate (188) - +- Project (187) - +- ShuffledHashJoin Inner BuildRight (186) - :- Exchange (181) - : +- Project (180) - : +- ShuffledHashJoin Inner BuildRight (179) - : :- Exchange (175) - : : +- Project (174) - : : +- ShuffledHashJoin Inner BuildRight (173) - : : :- Exchange (169) - : : : +- Project (168) - : : : +- ShuffledHashJoin Inner BuildRight (167) - : : : :- Exchange (163) - : : : : +- Project (162) - : : : : +- ShuffledHashJoin Inner BuildRight (161) - : : : : :- Exchange (157) - : : : : : +- Project (156) - : : : : : +- ShuffledHashJoin Inner BuildRight (155) - : : : : : :- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) - : : : : : : :- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Exchange (148) - : : : : : : +- Filter (147) - : : : : : : +- Scan parquet (146) - : : : : : +- Exchange (154) - : : : : : +- Filter (153) - : : : : : +- Scan parquet (152) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Exchange (166) - : : : +- Filter (165) - : : : +- Scan parquet (164) - : : +- Exchange (172) - : : +- Filter (171) - : : +- Scan parquet (170) - : +- Exchange (178) - : +- Filter (177) - : +- Scan parquet (176) - +- Exchange (185) - +- Project (184) - +- Filter (183) - +- Scan parquet (182) + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) (1) Scan parquet @@ -750,228 +764,284 @@ Input [2]: [p_partkey#X, p_type#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(147) Filter +(148) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(148) Exchange +(149) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(150) Project +(152) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(153) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(153) Filter +(156) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(154) Exchange +(157) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(156) Project +(160) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(157) Exchange +(161) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(159) Filter +(164) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(160) Exchange +(165) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(161) ShuffledHashJoin +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(162) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(163) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Scan parquet +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(165) Filter +(172) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(166) Exchange +(173) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(167) ShuffledHashJoin +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(168) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(169) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) Scan parquet +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(171) Filter +(180) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(172) Exchange +(181) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(173) ShuffledHashJoin +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(174) Project +(184) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(175) Exchange +(185) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Scan parquet +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(177) Filter +(188) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(178) Exchange +(189) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(179) ShuffledHashJoin +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(180) Project +(192) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(181) Exchange +(193) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(182) Scan parquet +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(183) Filter +(196) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(184) Project +(197) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(185) Exchange +(198) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(186) ShuffledHashJoin +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(187) Project +(201) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(188) HashAggregate +(202) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(189) Exchange +(203) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(190) HashAggregate +(204) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(191) Exchange +(205) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(206) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(193) AdaptiveSparkPlan +(207) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index d6bc308a9c2a..cb207c0800c3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (155) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,44 +83,54 @@ AdaptiveSparkPlan (145) +- ^ FilterExecTransformer (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (137) - +- Filter (136) - +- Scan parquet (135) + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -570,168 +580,208 @@ Input [2]: [p_partkey#X, p_name#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(113) Exchange +(114) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(115) Project +(117) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(118) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(119) Exchange +(122) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(121) Project +(125) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(122) Exchange +(126) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(125) Exchange +(130) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(127) Project +(133) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(128) Exchange +(134) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(131) Exchange +(138) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(133) Project +(141) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(134) Exchange +(142) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(137) Exchange +(146) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(139) Project +(149) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(140) HashAggregate +(150) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(141) Exchange +(151) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(152) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(143) Exchange +(153) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(154) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(155) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index fcb13291c838..c295515b8a6c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == VeloxColumnarToRowExec (67) +- TakeOrderedAndProjectExecTransformer (66) @@ -54,32 +54,38 @@ AdaptiveSparkPlan (94) +- ^ FilterExecTransformer (45) +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- ShuffledHashJoin Inner BuildRight (88) - :- Exchange (84) - : +- Project (83) - : +- ShuffledHashJoin Inner BuildRight (82) - : :- Exchange (77) - : : +- Project (76) - : : +- ShuffledHashJoin Inner BuildRight (75) - : : :- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (81) - : +- Project (80) - : +- Filter (79) - : +- Scan parquet (78) - +- Exchange (87) - +- Filter (86) - +- Scan parquet (85) + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -371,116 +377,140 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(74) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(75) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(76) Project +(78) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(77) Exchange +(79) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(79) Filter +(82) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(80) Project +(83) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(81) Exchange +(84) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(83) Project +(87) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(84) Exchange +(88) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(87) Exchange +(92) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) ShuffledHashJoin +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(89) Project +(95) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(90) HashAggregate +(96) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(91) Exchange +(97) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(98) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) TakeOrderedAndProject +(99) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index bbd20320b798..20bb486f3841 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (78) +AdaptiveSparkPlan (82) +- == Final Plan == VeloxColumnarToRowExec (56) +- ^ SortExecTransformer (54) @@ -45,27 +45,31 @@ AdaptiveSparkPlan (78) +- ^ FilterExecTransformer (28) +- ^ Scan parquet (27) +- == Initial Plan == - Sort (77) - +- Exchange (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Project (71) - +- ShuffledHashJoin Inner BuildRight (70) - :- Exchange (65) - : +- Project (64) - : +- ShuffledHashJoin Inner BuildRight (63) - : :- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Exchange (62) - : +- Filter (61) - : +- Scan parquet (60) - +- Exchange (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) (1) Scan parquet @@ -311,359 +315,395 @@ Condition : isnotnull(ps_suppkey#X) Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) Exchange +(63) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) ShuffledHashJoin +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(64) Project +(66) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(67) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Scan parquet +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(68) Project +(71) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(69) Exchange +(72) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) ShuffledHashJoin +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) Project +(75) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(72) HashAggregate +(76) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(73) Exchange +(77) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(78) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(75) Filter +(79) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(76) Exchange +(80) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Sort +(81) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(78) AdaptiveSparkPlan +(82) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (110) - +- ^ ProjectExecTransformer (108) - +- ^ RegularHashAggregateExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ ProjectExecTransformer (105) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - :- ^ InputIteratorTransformer (99) - : +- ShuffleQueryStage (97), Statistics(X) - : +- ColumnarExchange (96) - : +- VeloxAppendBatches (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (87) - : : +- ShuffleQueryStage (85), Statistics(X) - : : +- ColumnarExchange (84) - : : +- VeloxAppendBatches (83) - : : +- ^ ProjectExecTransformer (81) - : : +- ^ FilterExecTransformer (80) - : : +- ^ Scan parquet (79) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ReusedExchange (88) - +- ^ InputIteratorTransformer (103) - +- ShuffleQueryStage (101), Statistics(X) - +- ReusedExchange (100) + VeloxColumnarToRowExec (114) + +- ^ ProjectExecTransformer (112) + +- ^ RegularHashAggregateExecTransformer (111) + +- ^ RegularHashAggregateExecTransformer (110) + +- ^ ProjectExecTransformer (109) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) + :- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) + : :- ^ InputIteratorTransformer (91) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- ColumnarExchange (88) + : : +- VeloxAppendBatches (87) + : : +- ^ ProjectExecTransformer (85) + : : +- ^ FilterExecTransformer (84) + : : +- ^ Scan parquet (83) + : +- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ReusedExchange (92) + +- ^ InputIteratorTransformer (107) + +- ShuffleQueryStage (105), Statistics(X) + +- ReusedExchange (104) +- == Initial Plan == - HashAggregate (127) - +- HashAggregate (126) - +- Project (125) - +- ShuffledHashJoin Inner BuildRight (124) - :- Exchange (119) - : +- Project (118) - : +- ShuffledHashJoin Inner BuildRight (117) - : :- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- Exchange (123) - +- Project (122) - +- Filter (121) - +- Scan parquet (120) - - -(79) Scan parquet + HashAggregate (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Filter (116) + : : +- Scan parquet (115) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) + + +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) FilterExecTransformer +(84) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(81) ProjectExecTransformer +(85) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(82) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(83) VeloxAppendBatches +(87) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(88) ReusedExchange [Reuses operator id: 15] +(92) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(94) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(95) VeloxAppendBatches +(99) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(96) ColumnarExchange +(100) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(97) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(98) InputAdapter +(102) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(100) ReusedExchange [Reuses operator id: 32] +(104) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(101) ShuffleQueryStage +(105) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(102) InputAdapter +(106) InputAdapter Input [1]: [n_nationkey#X] -(103) InputIteratorTransformer +(107) InputIteratorTransformer Input [1]: [n_nationkey#X] -(104) ShuffledHashJoinExecTransformer +(108) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(105) ProjectExecTransformer +(109) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(106) RegularHashAggregateExecTransformer +(110) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(107) RegularHashAggregateExecTransformer +(111) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(108) ProjectExecTransformer +(112) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(109) WholeStageCodegenTransformer (X) +(113) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(110) VeloxColumnarToRowExec +(114) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(111) Scan parquet +(115) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(112) Filter +(116) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(113) Exchange +(117) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) Scan parquet +(118) Sort +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(119) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(115) Filter +(120) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(116) Exchange +(121) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(118) Project +(124) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(119) Exchange +(125) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Scan parquet +(126) Sort +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(121) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(122) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(123) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) ShuffledHashJoin +(131) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(125) Project +(133) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(126) HashAggregate +(134) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(127) HashAggregate +(135) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(128) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index 194b60bb7713..1b36d274aab4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,20 +31,22 @@ AdaptiveSparkPlan (53) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin Inner BuildLeft (46) - :- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -218,60 +220,68 @@ Condition : isnotnull(o_orderkey#X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(44) Project +(45) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(45) Exchange +(46) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(49) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(51) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(53) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index a9d4e199cfd2..83ec9aeda98a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (58) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ SortExecTransformer (38) @@ -33,21 +33,23 @@ AdaptiveSparkPlan (56) +- ^ FilterExecTransformer (10) +- ^ Scan parquet (9) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftOuter BuildRight (47) - :- Exchange (42) - : +- Scan parquet (41) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftOuter (49) + :- Sort (43) + : +- Exchange (42) + : +- Scan parquet (41) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -229,74 +231,82 @@ ReadSchema: struct Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(45) Project +(46) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(46) Exchange +(47) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(50) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(51) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(53) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(54) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(55) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(56) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(57) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt index afac58cb52bc..a1108606b5bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (39) +- == Final Plan == VeloxColumnarToRowExec (25) +- ^ ProjectExecTransformer (23) @@ -22,17 +22,19 @@ AdaptiveSparkPlan (37) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- ShuffledHashJoin Inner BuildRight (33) - :- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -162,44 +164,52 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Scan parquet +(30) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(31) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(32) Exchange +(33) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) ShuffledHashJoin +(34) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) Project +(36) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(36) HashAggregate +(38) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(37) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index fec017400c11..88730deb3c32 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == VeloxColumnarToRowExec (30) +- AQEShuffleRead (29) @@ -27,20 +27,22 @@ AdaptiveSparkPlan (45) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -186,221 +188,229 @@ Condition : isnotnull(s_suppkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(34) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(40) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(42) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (71) +AdaptiveSparkPlan (73) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ ProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- VeloxAppendBatches (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (64) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ ProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ ProjectExecTransformer (52) + +- ^ FlushableHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ FilterExecTransformer (49) + +- ^ Scan parquet (48) +- == Initial Plan == - HashAggregate (70) - +- HashAggregate (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (72) + +- HashAggregate (71) + +- HashAggregate (70) + +- Exchange (69) + +- HashAggregate (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) -(46) Scan parquet +(48) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(47) FilterExecTransformer +(49) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(48) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(49) FlushableHashAggregateExecTransformer +(51) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(52) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(52) VeloxAppendBatches +(54) VeloxAppendBatches Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) ColumnarExchange +(55) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(57) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) ProjectExecTransformer +(60) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(60) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(62) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(63) Scan parquet +(65) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(64) Filter +(66) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(65) Project +(67) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(67) Exchange +(69) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(70) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(69) HashAggregate +(71) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(70) HashAggregate +(72) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(71) AdaptiveSparkPlan +(73) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index 15dd2fa6da8e..535b6940301d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (71) +- == Final Plan == VeloxColumnarToRowExec (47) +- ^ SortExecTransformer (45) @@ -38,27 +38,29 @@ AdaptiveSparkPlan (69) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- ShuffledHashJoin Inner BuildRight (59) - :- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (58) - +- Filter (57) - +- Scan parquet (56) + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -296,74 +298,82 @@ Join condition: None Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(58) Exchange +(59) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) ShuffledHashJoin +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(60) Project +(62) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(61) HashAggregate +(63) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(65) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(67) Exchange +(69) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(70) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 69af0fd38e92..d360b6c948e3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (63) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ ProjectExecTransformer (38) @@ -35,25 +35,28 @@ AdaptiveSparkPlan (60) +- ^ FilterExecTransformer (22) +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (59) - +- HashAggregate (58) - +- Project (57) - +- ShuffledHashJoin Inner BuildRight (56) - :- Project (49) - : +- ShuffledHashJoin Inner BuildRight (48) - : :- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Filter (51) - +- Scan parquet (50) + HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (44) + : : +- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -247,90 +250,102 @@ Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(44) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(45) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(46) Project +(47) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(47) Exchange +(48) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(49) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(49) Project +(51) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(50) Scan parquet +(52) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(51) Filter +(53) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(52) HashAggregate +(54) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(53) Exchange +(55) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) Filter +(57) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(56) ShuffledHashJoin +(58) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(57) Project +(60) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(58) HashAggregate +(61) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(59) HashAggregate +(62) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(60) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index 8db05ed7572c..a664adfd3175 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (110) +- == Final Plan == VeloxColumnarToRowExec (70) +- TakeOrderedAndProjectExecTransformer (69) @@ -58,38 +58,45 @@ AdaptiveSparkPlan (103) +- ShuffleQueryStage (57), Statistics(X) +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (109) + +- HashAggregate (108) + +- HashAggregate (107) + +- Project (106) + +- SortMergeJoin Inner (105) + :- Sort (92) + : +- Exchange (91) + : +- Project (90) + : +- SortMergeJoin Inner (89) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (88) + : +- Exchange (87) + : +- SortMergeJoin LeftSemi (86) + : :- Sort (78) + : : +- Exchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Sort (85) + : +- Project (84) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- Scan parquet (79) + +- SortMergeJoin LeftSemi (104) + :- Sort (96) + : +- Exchange (95) + : +- Filter (94) + : +- Scan parquet (93) + +- Sort (103) + +- Project (102) + +- Filter (101) + +- HashAggregate (100) + +- Exchange (99) + +- HashAggregate (98) + +- Scan parquet (97) (1) Scan parquet @@ -401,154 +408,182 @@ Condition : isnotnull(c_custkey#X) Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(77) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(80) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(81) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(82) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(83) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(84) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(85) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(84) Exchange +(87) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(88) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(89) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(86) Project +(90) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(91) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(92) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(93) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(98) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(99) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(100) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(101) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(102) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(103) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) ShuffledHashJoin +(105) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(99) Project +(106) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(107) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(108) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(109) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt index 14a5515d1e79..58e80362020f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (36) +AdaptiveSparkPlan (38) +- == Final Plan == VeloxColumnarToRowExec (24) +- ^ RegularHashAggregateExecTransformer (22) @@ -21,17 +21,19 @@ AdaptiveSparkPlan (36) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (35) - +- HashAggregate (34) - +- Project (33) - +- ShuffledHashJoin Inner BuildRight (32) - :- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -157,44 +159,52 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(29) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(30) Filter +(31) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(31) Exchange +(32) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) ShuffledHashJoin +(33) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(33) Project +(35) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(34) HashAggregate +(36) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(36) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt index 7b840720bc90..c22b822e6f7d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (143) +- == Final Plan == VeloxColumnarToRowExec (93) +- AQEShuffleRead (92) @@ -75,45 +75,55 @@ AdaptiveSparkPlan (133) +- ^ FilterExecTransformer (78) +- ^ Scan parquet (77) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- Project (130) - +- ShuffledHashJoin Inner BuildRight (129) - :- Exchange (124) - : +- Project (123) - : +- ShuffledHashJoin LeftSemi BuildRight (122) - : :- Exchange (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (121) - : +- Project (120) - : +- ShuffledHashJoin Inner BuildLeft (119) - : :- Exchange (105) - : : +- ShuffledHashJoin LeftSemi BuildRight (104) - : : :- Exchange (99) - : : : +- Filter (98) - : : : +- Scan parquet (97) - : : +- Exchange (103) - : : +- Project (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- Exchange (118) - : +- Filter (117) - : +- HashAggregate (116) - : +- HashAggregate (115) - : +- ShuffledHashJoin LeftSemi BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (128) - +- Project (127) - +- Filter (126) - +- Scan parquet (125) + Sort (142) + +- Exchange (141) + +- Project (140) + +- SortMergeJoin Inner (139) + :- Sort (133) + : +- Exchange (132) + : +- Project (131) + : +- SortMergeJoin LeftSemi (130) + : :- Sort (97) + : : +- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Sort (129) + : +- Exchange (128) + : +- Project (127) + : +- SortMergeJoin Inner (126) + : :- Sort (109) + : : +- Exchange (108) + : : +- SortMergeJoin LeftSemi (107) + : : :- Sort (101) + : : : +- Exchange (100) + : : : +- Filter (99) + : : : +- Scan parquet (98) + : : +- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Sort (125) + : +- Exchange (124) + : +- Filter (123) + : +- HashAggregate (122) + : +- HashAggregate (121) + : +- SortMergeJoin LeftSemi (120) + : :- Sort (114) + : : +- Exchange (113) + : : +- Project (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Sort (119) + : +- Exchange (118) + : +- Project (117) + : +- Filter (116) + : +- Scan parquet (115) + +- Sort (138) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -507,176 +517,216 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Scan parquet +(97) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(98) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(98) Filter +(99) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(99) Exchange +(100) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(101) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(102) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(103) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(104) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(105) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(106) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(107) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(109) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(110) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(107) Filter +(111) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(108) Project +(112) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(109) Exchange +(113) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(114) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(115) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(112) Project +(117) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(113) Exchange +(118) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(120) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(115) HashAggregate +(121) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(116) HashAggregate +(122) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(117) Filter +(123) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(118) Exchange +(124) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(125) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(120) Project +(127) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(121) Exchange +(128) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(129) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(130) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(123) Project +(131) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(124) Exchange +(132) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Scan parquet +(133) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(126) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(127) Project +(136) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(128) Exchange +(137) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) ShuffledHashJoin +(138) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(139) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(130) Project +(140) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(131) Exchange +(141) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(142) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(133) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index 5c05ec24757e..8413e2f8f232 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (137) +- == Final Plan == VeloxColumnarToRowExec (91) +- ^ RegularHashAggregateExecTransformer (89) @@ -72,42 +72,51 @@ AdaptiveSparkPlan (128) +- ^ FilterExecTransformer (71) +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildLeft (108) - : : :- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Exchange (107) - : : +- ShuffledHashJoin LeftAnti BuildRight (106) - : : :- ShuffledHashJoin LeftSemi BuildRight (101) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Exchange (100) - : : : +- Scan parquet (99) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Exchange (114) - : +- Project (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + TakeOrderedAndProject (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (116) + : : +- Exchange (115) + : : +- Project (114) + : : +- SortMergeJoin Inner (113) + : : :- Sort (95) + : : : +- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftAnti (110) + : : :- SortMergeJoin LeftSemi (104) + : : : :- Sort (100) + : : : : +- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (103) + : : : +- Exchange (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) (1) Scan parquet @@ -496,163 +505,199 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(95) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(100) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(100) Exchange +(102) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(103) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(102) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(103) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(104) Project +(107) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(105) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(109) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(107) Exchange +(111) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(112) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(114) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(110) Exchange +(115) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(116) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(113) Project +(119) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(114) Exchange +(120) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(121) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(116) Project +(123) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(117) Exchange +(124) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(125) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(120) Project +(128) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(129) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(132) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(124) HashAggregate +(133) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(125) Exchange +(134) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(135) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(127) TakeOrderedAndProject +(136) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(128) AdaptiveSparkPlan +(137) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index af5c086c274a..214b34066a8f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (52) +- == Final Plan == VeloxColumnarToRowExec (37) +- ^ SortExecTransformer (35) @@ -30,18 +30,20 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftAnti BuildRight (43) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (42) - +- Scan parquet (41) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) (1) Scan parquet @@ -210,191 +212,199 @@ Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23 Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(42) Exchange +(43) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(46) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(48) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(50) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(51) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (70) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ InputIteratorTransformer (60) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- VeloxAppendBatches (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ Scan parquet (53) +- == Initial Plan == - HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) -(51) Scan parquet +(53) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) ProjectExecTransformer +(55) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) FlushableHashAggregateExecTransformer +(56) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(56) VeloxAppendBatches +(58) VeloxAppendBatches Input [2]: [sum#X, count#X] Arguments: X -(57) ColumnarExchange +(59) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [2]: [sum#X, count#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [sum#X, count#X] -(61) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(62) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(63) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(64) Scan parquet +(66) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(65) Filter +(67) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(66) Project +(68) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(68) Exchange +(70) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(69) HashAggregate +(71) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(70) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (70) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ InputIteratorTransformer (60) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- VeloxAppendBatches (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ Scan parquet (53) +- == Initial Plan == - HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) \ No newline at end of file + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index 51408f03e4e7..df17819cafe9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == VeloxColumnarToRowExec (43) +- TakeOrderedAndProjectExecTransformer (42) @@ -36,25 +36,29 @@ AdaptiveSparkPlan (63) +- ^ FilterExecTransformer (28) +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- ShuffledHashJoin Inner BuildRight (58) - :- Exchange (53) - : +- Project (52) - : +- ShuffledHashJoin Inner BuildLeft (51) - : :- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Project (63) + +- SortMergeJoin Inner (62) + :- Sort (56) + : +- Exchange (55) + : +- Project (54) + : +- SortMergeJoin Inner (53) + : :- Sort (48) + : : +- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Sort (52) + : +- Exchange (51) + : +- Filter (50) + : +- Scan parquet (49) + +- Sort (61) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -256,80 +260,96 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Scan parquet +(48) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(49) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(49) Filter +(50) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(50) Exchange +(51) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(52) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(53) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(52) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(53) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(56) Project +(59) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(57) Exchange +(60) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) ShuffledHashJoin +(61) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(62) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) Project +(63) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(60) HashAggregate +(64) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(61) HashAggregate +(65) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index 4da32d7f70ac..85d303df874f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (56) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,21 +31,23 @@ AdaptiveSparkPlan (54) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftSemi BuildRight (47) - :- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -223,60 +225,68 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(45) Project +(46) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(46) Exchange +(47) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(50) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(51) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(54) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(55) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 2669a9fce3ae..8978f9563c68 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (156) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,45 +83,55 @@ AdaptiveSparkPlan (146) +- ^ FilterExecTransformer (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- HashAggregate (143) - +- Exchange (142) - +- HashAggregate (141) - +- Project (140) - +- ShuffledHashJoin Inner BuildRight (139) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Project (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (138) - +- Project (137) - +- Filter (136) - +- Scan parquet (135) + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -567,176 +577,216 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(113) Exchange +(114) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(115) Project +(117) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(118) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(119) Exchange +(122) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(121) Project +(125) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(122) Exchange +(126) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(125) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(127) Project +(133) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(128) Exchange +(134) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(131) Exchange +(138) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(141) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(134) Exchange +(142) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(137) Project +(146) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(138) Exchange +(147) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(140) Project +(150) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(141) HashAggregate +(151) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(142) Exchange +(152) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) HashAggregate +(153) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(144) Exchange +(154) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(155) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(146) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index b5abf7e36164..244f650f3a72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (139) +AdaptiveSparkPlan (149) +- == Final Plan == VeloxColumnarToRowExec (101) +- ^ SortExecTransformer (99) @@ -79,43 +79,53 @@ AdaptiveSparkPlan (139) +- ShuffleQueryStage (79), Statistics(X) +- ReusedExchange (78) +- == Initial Plan == - Sort (138) - +- Exchange (137) - +- HashAggregate (136) - +- Exchange (135) - +- HashAggregate (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (128) - : +- Project (127) - : +- ShuffledHashJoin Inner BuildRight (126) - : :- Exchange (122) - : : +- Project (121) - : : +- ShuffledHashJoin Inner BuildRight (120) - : : :- Exchange (116) - : : : +- Project (115) - : : : +- ShuffledHashJoin Inner BuildRight (114) - : : : :- Exchange (110) - : : : : +- Project (109) - : : : : +- ShuffledHashJoin Inner BuildLeft (108) - : : : : :- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Exchange (113) - : : : +- Filter (112) - : : : +- Scan parquet (111) - : : +- Exchange (119) - : : +- Filter (118) - : : +- Scan parquet (117) - : +- Exchange (125) - : +- Filter (124) - : +- Scan parquet (123) - +- Exchange (131) - +- Filter (130) - +- Scan parquet (129) + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) (1) Scan parquet @@ -537,168 +547,208 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(107) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(108) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(109) Project +(111) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(110) Exchange +(112) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(112) Filter +(115) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(113) Exchange +(116) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(115) Project +(119) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(120) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(118) Filter +(123) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(119) Exchange +(124) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(121) Project +(127) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(122) Exchange +(128) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(124) Filter +(131) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(125) Exchange +(132) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(135) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(128) Exchange +(136) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(130) Filter +(139) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(131) Exchange +(140) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(133) Project +(143) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(134) HashAggregate +(144) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(135) Exchange +(145) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) HashAggregate +(146) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(137) Exchange +(147) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(148) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(139) AdaptiveSparkPlan +(149) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index 47886e292bf7..282790ba6507 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (193) +AdaptiveSparkPlan (207) +- == Final Plan == VeloxColumnarToRowExec (141) +- ^ SortExecTransformer (139) @@ -110,57 +110,71 @@ AdaptiveSparkPlan (193) +- ^ FilterExecTransformer (113) +- ^ Scan parquet (112) +- == Initial Plan == - Sort (192) - +- Exchange (191) - +- HashAggregate (190) - +- Exchange (189) - +- HashAggregate (188) - +- Project (187) - +- ShuffledHashJoin Inner BuildRight (186) - :- Exchange (181) - : +- Project (180) - : +- ShuffledHashJoin Inner BuildRight (179) - : :- Exchange (175) - : : +- Project (174) - : : +- ShuffledHashJoin Inner BuildRight (173) - : : :- Exchange (169) - : : : +- Project (168) - : : : +- ShuffledHashJoin Inner BuildRight (167) - : : : :- Exchange (163) - : : : : +- Project (162) - : : : : +- ShuffledHashJoin Inner BuildRight (161) - : : : : :- Exchange (157) - : : : : : +- Project (156) - : : : : : +- ShuffledHashJoin Inner BuildRight (155) - : : : : : :- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) - : : : : : : :- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Exchange (148) - : : : : : : +- Filter (147) - : : : : : : +- Scan parquet (146) - : : : : : +- Exchange (154) - : : : : : +- Filter (153) - : : : : : +- Scan parquet (152) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Exchange (166) - : : : +- Filter (165) - : : : +- Scan parquet (164) - : : +- Exchange (172) - : : +- Filter (171) - : : +- Scan parquet (170) - : +- Exchange (178) - : +- Filter (177) - : +- Scan parquet (176) - +- Exchange (185) - +- Project (184) - +- Filter (183) - +- Scan parquet (182) + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) (1) Scan parquet @@ -750,228 +764,284 @@ Input [2]: [p_partkey#X, p_type#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(147) Filter +(148) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(148) Exchange +(149) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(150) Project +(152) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(153) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(153) Filter +(156) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(154) Exchange +(157) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(156) Project +(160) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(157) Exchange +(161) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(159) Filter +(164) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(160) Exchange +(165) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(161) ShuffledHashJoin +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(162) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(163) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Scan parquet +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(165) Filter +(172) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(166) Exchange +(173) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(167) ShuffledHashJoin +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(168) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(169) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) Scan parquet +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(171) Filter +(180) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(172) Exchange +(181) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(173) ShuffledHashJoin +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(174) Project +(184) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(175) Exchange +(185) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Scan parquet +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(177) Filter +(188) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(178) Exchange +(189) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(179) ShuffledHashJoin +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(180) Project +(192) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(181) Exchange +(193) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(182) Scan parquet +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(183) Filter +(196) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(184) Project +(197) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(185) Exchange +(198) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(186) ShuffledHashJoin +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(187) Project +(201) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(188) HashAggregate +(202) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(189) Exchange +(203) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(190) HashAggregate +(204) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(191) Exchange +(205) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(206) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(193) AdaptiveSparkPlan +(207) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 0bad5c20cf05..15fbf97a77f3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (155) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,44 +83,54 @@ AdaptiveSparkPlan (145) +- ^ FilterExecTransformer (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (137) - +- Filter (136) - +- Scan parquet (135) + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -570,168 +580,208 @@ Input [2]: [p_partkey#X, p_name#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(113) Exchange +(114) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(115) Project +(117) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(118) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(119) Exchange +(122) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(121) Project +(125) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(122) Exchange +(126) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(125) Exchange +(130) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(127) Project +(133) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(128) Exchange +(134) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(131) Exchange +(138) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(133) Project +(141) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(134) Exchange +(142) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(137) Exchange +(146) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(139) Project +(149) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(140) HashAggregate +(150) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(141) Exchange +(151) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(152) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(143) Exchange +(153) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(154) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(155) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 5be72ee42483..d7376c740f93 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (94) +AdaptiveSparkPlan (100) +- == Final Plan == VeloxColumnarToRowExec (67) +- TakeOrderedAndProjectExecTransformer (66) @@ -54,32 +54,38 @@ AdaptiveSparkPlan (94) +- ^ FilterExecTransformer (45) +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (93) - +- HashAggregate (92) - +- Exchange (91) - +- HashAggregate (90) - +- Project (89) - +- ShuffledHashJoin Inner BuildRight (88) - :- Exchange (84) - : +- Project (83) - : +- ShuffledHashJoin Inner BuildRight (82) - : :- Exchange (77) - : : +- Project (76) - : : +- ShuffledHashJoin Inner BuildRight (75) - : : :- Exchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (81) - : +- Project (80) - : +- Filter (79) - : +- Scan parquet (78) - +- Exchange (87) - +- Filter (86) - +- Scan parquet (85) + TakeOrderedAndProject (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- SortMergeJoin Inner (94) + :- Sort (89) + : +- Exchange (88) + : +- Project (87) + : +- SortMergeJoin Inner (86) + : :- Sort (80) + : : +- Exchange (79) + : : +- Project (78) + : : +- SortMergeJoin Inner (77) + : : :- Sort (71) + : : : +- Exchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- Sort (76) + : : +- Exchange (75) + : : +- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- Sort (85) + : +- Exchange (84) + : +- Project (83) + : +- Filter (82) + : +- Scan parquet (81) + +- Sort (93) + +- Exchange (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -374,119 +380,143 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(71) Sort +Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(72) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(74) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(75) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(76) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(77) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(76) Project +(78) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(77) Exchange +(79) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(80) Sort +Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(81) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(79) Filter +(82) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(80) Project +(83) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(81) Exchange +(84) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(85) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(83) Project +(87) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(84) Exchange +(88) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(89) Sort +Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(90) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(86) Filter +(91) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(87) Exchange +(92) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) ShuffledHashJoin +(93) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(94) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(89) Project +(95) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(90) HashAggregate +(96) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(91) Exchange +(97) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(98) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) TakeOrderedAndProject +(99) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(94) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index d9dbbfe0dbe9..c9371ffbf2c3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (78) +AdaptiveSparkPlan (82) +- == Final Plan == VeloxColumnarToRowExec (56) +- ^ SortExecTransformer (54) @@ -45,27 +45,31 @@ AdaptiveSparkPlan (78) +- ^ FilterExecTransformer (28) +- ^ Scan parquet (27) +- == Initial Plan == - Sort (77) - +- Exchange (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Project (71) - +- ShuffledHashJoin Inner BuildRight (70) - :- Exchange (65) - : +- Project (64) - : +- ShuffledHashJoin Inner BuildRight (63) - : :- Exchange (59) - : : +- Filter (58) - : : +- Scan parquet (57) - : +- Exchange (62) - : +- Filter (61) - : +- Scan parquet (60) - +- Exchange (69) - +- Project (68) - +- Filter (67) - +- Scan parquet (66) + Sort (81) + +- Exchange (80) + +- Filter (79) + +- HashAggregate (78) + +- Exchange (77) + +- HashAggregate (76) + +- Project (75) + +- SortMergeJoin Inner (74) + :- Sort (68) + : +- Exchange (67) + : +- Project (66) + : +- SortMergeJoin Inner (65) + : :- Sort (60) + : : +- Exchange (59) + : : +- Filter (58) + : : +- Scan parquet (57) + : +- Sort (64) + : +- Exchange (63) + : +- Filter (62) + : +- Scan parquet (61) + +- Sort (73) + +- Exchange (72) + +- Project (71) + +- Filter (70) + +- Scan parquet (69) (1) Scan parquet @@ -313,365 +317,401 @@ Condition : isnotnull(ps_suppkey#X) Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(60) Sort +Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(61) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) Exchange +(63) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) ShuffledHashJoin +(64) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(65) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(64) Project +(66) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(67) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Scan parquet +(68) Sort +Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(67) Filter +(70) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(68) Project +(71) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(69) Exchange +(72) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) ShuffledHashJoin +(73) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(74) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) Project +(75) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(72) HashAggregate +(76) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(73) Exchange +(77) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(78) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(75) Filter +(79) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(76) Exchange +(80) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Sort +(81) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(78) AdaptiveSparkPlan +(82) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (110) - +- ^ ProjectExecTransformer (108) - +- ^ RegularHashAggregateExecTransformer (107) - +- ^ RegularHashAggregateExecTransformer (106) - +- ^ ProjectExecTransformer (105) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - :- ^ InputIteratorTransformer (99) - : +- ShuffleQueryStage (97), Statistics(X) - : +- ColumnarExchange (96) - : +- VeloxAppendBatches (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (87) - : : +- ShuffleQueryStage (85), Statistics(X) - : : +- ColumnarExchange (84) - : : +- VeloxAppendBatches (83) - : : +- ^ ProjectExecTransformer (81) - : : +- ^ FilterExecTransformer (80) - : : +- ^ Scan parquet (79) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ReusedExchange (88) - +- ^ InputIteratorTransformer (103) - +- ShuffleQueryStage (101), Statistics(X) - +- ReusedExchange (100) + VeloxColumnarToRowExec (114) + +- ^ ProjectExecTransformer (112) + +- ^ RegularHashAggregateExecTransformer (111) + +- ^ RegularHashAggregateExecTransformer (110) + +- ^ ProjectExecTransformer (109) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) + :- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) + : :- ^ InputIteratorTransformer (91) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- ColumnarExchange (88) + : : +- VeloxAppendBatches (87) + : : +- ^ ProjectExecTransformer (85) + : : +- ^ FilterExecTransformer (84) + : : +- ^ Scan parquet (83) + : +- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ReusedExchange (92) + +- ^ InputIteratorTransformer (107) + +- ShuffleQueryStage (105), Statistics(X) + +- ReusedExchange (104) +- == Initial Plan == - HashAggregate (127) - +- HashAggregate (126) - +- Project (125) - +- ShuffledHashJoin Inner BuildRight (124) - :- Exchange (119) - : +- Project (118) - : +- ShuffledHashJoin Inner BuildRight (117) - : :- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (116) - : +- Filter (115) - : +- Scan parquet (114) - +- Exchange (123) - +- Project (122) - +- Filter (121) - +- Scan parquet (120) - - -(79) Scan parquet + HashAggregate (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Filter (116) + : : +- Scan parquet (115) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) + + +(83) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(80) FilterExecTransformer +(84) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(81) ProjectExecTransformer +(85) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(82) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(83) VeloxAppendBatches +(87) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(84) ColumnarExchange +(88) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(86) InputAdapter +(90) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(87) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(88) ReusedExchange [Reuses operator id: 15] +(92) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(89) ShuffleQueryStage +(93) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(94) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(91) InputIteratorTransformer +(95) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(92) ShuffledHashJoinExecTransformer +(96) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(97) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(94) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(95) VeloxAppendBatches +(99) VeloxAppendBatches Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(96) ColumnarExchange +(100) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(97) ShuffleQueryStage +(101) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(98) InputAdapter +(102) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(99) InputIteratorTransformer +(103) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(100) ReusedExchange [Reuses operator id: 32] +(104) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(101) ShuffleQueryStage +(105) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(102) InputAdapter +(106) InputAdapter Input [1]: [n_nationkey#X] -(103) InputIteratorTransformer +(107) InputIteratorTransformer Input [1]: [n_nationkey#X] -(104) ShuffledHashJoinExecTransformer +(108) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(105) ProjectExecTransformer +(109) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(106) RegularHashAggregateExecTransformer +(110) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(107) RegularHashAggregateExecTransformer +(111) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(108) ProjectExecTransformer +(112) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(109) WholeStageCodegenTransformer (X) +(113) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(110) VeloxColumnarToRowExec +(114) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(111) Scan parquet +(115) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(112) Filter +(116) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(113) Exchange +(117) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) Scan parquet +(118) Sort +Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(119) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(115) Filter +(120) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(116) Exchange +(121) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) ShuffledHashJoin +(122) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(123) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(118) Project +(124) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(119) Exchange +(125) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Scan parquet +(126) Sort +Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(121) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(122) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(123) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) ShuffledHashJoin +(131) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(125) Project +(133) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(126) HashAggregate +(134) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(127) HashAggregate +(135) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(128) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 63c356d6d1bf..ce033f5468d1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (55) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,20 +31,22 @@ AdaptiveSparkPlan (53) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- ShuffledHashJoin Inner BuildLeft (46) - :- Exchange (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- SortMergeJoin Inner (48) + :- Sort (42) + : +- Exchange (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (47) + +- Exchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -219,61 +221,69 @@ Condition : isnotnull(o_orderkey#X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Scan parquet +(42) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(43) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(43) Filter +(44) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(44) Project +(45) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(45) Exchange +(46) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) ShuffledHashJoin +(47) Sort +Input [2]: [l_orderkey#X, l_shipmode#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(48) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(47) Project +(49) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(51) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(52) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(53) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(54) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index 812a7be868b6..c71d03b93e12 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (58) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ SortExecTransformer (38) @@ -33,21 +33,23 @@ AdaptiveSparkPlan (56) +- ^ FilterExecTransformer (10) +- ^ Scan parquet (9) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftOuter BuildRight (47) - :- Exchange (42) - : +- Scan parquet (41) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (57) + +- Exchange (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftOuter (49) + :- Sort (43) + : +- Exchange (42) + : +- Scan parquet (41) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -230,75 +232,83 @@ ReadSchema: struct Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(45) Project +(46) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(46) Exchange +(47) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(48) Project +(50) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(51) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(52) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(53) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(54) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(55) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(56) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(57) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt index c6f425f00868..492d3f8b9d07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (37) +AdaptiveSparkPlan (39) +- == Final Plan == VeloxColumnarToRowExec (25) +- ^ ProjectExecTransformer (23) @@ -22,17 +22,19 @@ AdaptiveSparkPlan (37) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (36) - +- HashAggregate (35) - +- Project (34) - +- ShuffledHashJoin Inner BuildRight (33) - :- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -163,45 +165,53 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Scan parquet +(30) Sort +Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(31) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(32) Exchange +(33) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) ShuffledHashJoin +(34) Sort +Input [2]: [p_partkey#X, p_type#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(34) Project +(36) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(35) HashAggregate +(37) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(36) HashAggregate +(38) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(37) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index e30eec3d854f..129e4ad927e9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == VeloxColumnarToRowExec (30) +- AQEShuffleRead (29) @@ -27,20 +27,22 @@ AdaptiveSparkPlan (45) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -187,222 +189,230 @@ Condition : isnotnull(s_suppkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(34) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(40) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(42) Sort +Input [2]: [supplier_no#X, total_revenue#X] +Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 22 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (71) +AdaptiveSparkPlan (73) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ ProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- VeloxAppendBatches (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (64) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ ProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ ProjectExecTransformer (52) + +- ^ FlushableHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ FilterExecTransformer (49) + +- ^ Scan parquet (48) +- == Initial Plan == - HashAggregate (70) - +- HashAggregate (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (72) + +- HashAggregate (71) + +- HashAggregate (70) + +- Exchange (69) + +- HashAggregate (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) -(46) Scan parquet +(48) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(47) FilterExecTransformer +(49) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(48) ProjectExecTransformer +(50) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(49) FlushableHashAggregateExecTransformer +(51) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(52) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(52) VeloxAppendBatches +(54) VeloxAppendBatches Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(53) ColumnarExchange +(55) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(57) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) ProjectExecTransformer +(60) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(59) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(60) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(61) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(62) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(63) Scan parquet +(65) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(64) Filter +(66) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(65) Project +(67) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(66) HashAggregate +(68) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(67) Exchange +(69) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(70) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(69) HashAggregate +(71) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(70) HashAggregate +(72) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(71) AdaptiveSparkPlan +(73) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index eff7577281e6..45b6041f8b4b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (71) +- == Final Plan == VeloxColumnarToRowExec (47) +- ^ SortExecTransformer (45) @@ -38,27 +38,29 @@ AdaptiveSparkPlan (69) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (68) - +- Exchange (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- ShuffledHashJoin Inner BuildRight (59) - :- Exchange (55) - : +- BroadcastHashJoin LeftAnti BuildRight (54) - : :- Filter (49) - : : +- Scan parquet (48) - : +- BroadcastExchange (53) - : +- Project (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (58) - +- Filter (57) - +- Scan parquet (56) + Sort (70) + +- Exchange (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- SortMergeJoin Inner (61) + :- Sort (56) + : +- Exchange (55) + : +- BroadcastHashJoin LeftAnti BuildRight (54) + : :- Filter (49) + : : +- Scan parquet (48) + : +- BroadcastExchange (53) + : +- Project (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Sort (60) + +- Exchange (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -298,75 +300,83 @@ Join condition: None Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(56) Sort +Input [2]: [ps_partkey#X, ps_suppkey#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(57) Filter +(58) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(58) Exchange +(59) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(59) ShuffledHashJoin +(60) Sort +Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(61) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(60) Project +(62) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(61) HashAggregate +(63) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(62) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(65) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(64) HashAggregate +(66) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(65) Exchange +(67) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(68) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(67) Exchange +(69) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Sort +(70) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(69) AdaptiveSparkPlan +(71) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index 649bfcbe40e1..b46b3e3f2724 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (63) +- == Final Plan == VeloxColumnarToRowExec (40) +- ^ ProjectExecTransformer (38) @@ -35,25 +35,28 @@ AdaptiveSparkPlan (60) +- ^ FilterExecTransformer (22) +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (59) - +- HashAggregate (58) - +- Project (57) - +- ShuffledHashJoin Inner BuildRight (56) - :- Project (49) - : +- ShuffledHashJoin Inner BuildRight (48) - : :- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Project (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Filter (51) - +- Scan parquet (50) + HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (44) + : : +- Exchange (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (58) + +- Filter (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -249,92 +252,104 @@ Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(44) Sort +Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(45) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(46) Project +(47) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(47) Exchange +(48) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(49) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(49) Project +(51) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(50) Scan parquet +(52) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(51) Filter +(53) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(52) HashAggregate +(54) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(53) Exchange +(55) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(56) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) Filter +(57) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(56) ShuffledHashJoin +(58) Sort +Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(57) Project +(60) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(58) HashAggregate +(61) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(59) HashAggregate +(62) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(60) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index c3075e511782..febb48962446 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (110) +- == Final Plan == VeloxColumnarToRowExec (70) +- TakeOrderedAndProjectExecTransformer (69) @@ -58,38 +58,45 @@ AdaptiveSparkPlan (103) +- ShuffleQueryStage (57), Statistics(X) +- ReusedExchange (56) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (109) + +- HashAggregate (108) + +- HashAggregate (107) + +- Project (106) + +- SortMergeJoin Inner (105) + :- Sort (92) + : +- Exchange (91) + : +- Project (90) + : +- SortMergeJoin Inner (89) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (88) + : +- Exchange (87) + : +- SortMergeJoin LeftSemi (86) + : :- Sort (78) + : : +- Exchange (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Sort (85) + : +- Project (84) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- Scan parquet (79) + +- SortMergeJoin LeftSemi (104) + :- Sort (96) + : +- Exchange (95) + : +- Filter (94) + : +- Scan parquet (93) + +- Sort (103) + +- Project (102) + +- Filter (101) + +- HashAggregate (100) + +- Exchange (99) + +- HashAggregate (98) + +- Scan parquet (97) (1) Scan parquet @@ -405,158 +412,186 @@ Condition : isnotnull(c_custkey#X) Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(74) Sort +Input [2]: [c_custkey#X, c_name#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(75) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(77) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(78) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(80) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(81) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(82) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(83) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(84) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(85) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(84) Exchange +(87) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(88) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(89) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(86) Project +(90) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(91) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(92) Sort +Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(93) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Sort +Input [2]: [l_orderkey#X, l_quantity#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(97) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(98) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(99) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(100) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(101) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(102) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(103) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(98) ShuffledHashJoin +(105) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(99) Project +(106) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(107) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(108) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(109) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(110) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt index baf4b2a51607..fa78645313e4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (36) +AdaptiveSparkPlan (38) +- == Final Plan == VeloxColumnarToRowExec (24) +- ^ RegularHashAggregateExecTransformer (22) @@ -21,17 +21,19 @@ AdaptiveSparkPlan (36) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (35) - +- HashAggregate (34) - +- Project (33) - +- ShuffledHashJoin Inner BuildRight (32) - :- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Exchange (31) - +- Filter (30) - +- Scan parquet (29) + HashAggregate (37) + +- HashAggregate (36) + +- Project (35) + +- SortMergeJoin Inner (34) + :- Sort (29) + : +- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Sort (33) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -158,45 +160,53 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Scan parquet +(29) Sort +Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(30) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(30) Filter +(31) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(31) Exchange +(32) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(32) ShuffledHashJoin +(33) Sort +Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(34) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(33) Project +(35) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(34) HashAggregate +(36) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(35) HashAggregate +(37) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(36) AdaptiveSparkPlan +(38) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt index 7ddecfe855eb..bb9987fc32c1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (143) +- == Final Plan == VeloxColumnarToRowExec (93) +- AQEShuffleRead (92) @@ -75,45 +75,55 @@ AdaptiveSparkPlan (133) +- ^ FilterExecTransformer (78) +- ^ Scan parquet (77) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- Project (130) - +- ShuffledHashJoin Inner BuildRight (129) - :- Exchange (124) - : +- Project (123) - : +- ShuffledHashJoin LeftSemi BuildRight (122) - : :- Exchange (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (121) - : +- Project (120) - : +- ShuffledHashJoin Inner BuildLeft (119) - : :- Exchange (105) - : : +- ShuffledHashJoin LeftSemi BuildRight (104) - : : :- Exchange (99) - : : : +- Filter (98) - : : : +- Scan parquet (97) - : : +- Exchange (103) - : : +- Project (102) - : : +- Filter (101) - : : +- Scan parquet (100) - : +- Exchange (118) - : +- Filter (117) - : +- HashAggregate (116) - : +- HashAggregate (115) - : +- ShuffledHashJoin LeftSemi BuildRight (114) - : :- Exchange (109) - : : +- Project (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (113) - : +- Project (112) - : +- Filter (111) - : +- Scan parquet (110) - +- Exchange (128) - +- Project (127) - +- Filter (126) - +- Scan parquet (125) + Sort (142) + +- Exchange (141) + +- Project (140) + +- SortMergeJoin Inner (139) + :- Sort (133) + : +- Exchange (132) + : +- Project (131) + : +- SortMergeJoin LeftSemi (130) + : :- Sort (97) + : : +- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Sort (129) + : +- Exchange (128) + : +- Project (127) + : +- SortMergeJoin Inner (126) + : :- Sort (109) + : : +- Exchange (108) + : : +- SortMergeJoin LeftSemi (107) + : : :- Sort (101) + : : : +- Exchange (100) + : : : +- Filter (99) + : : : +- Scan parquet (98) + : : +- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Sort (125) + : +- Exchange (124) + : +- Filter (123) + : +- HashAggregate (122) + : +- HashAggregate (121) + : +- SortMergeJoin LeftSemi (120) + : :- Sort (114) + : : +- Exchange (113) + : : +- Project (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Sort (119) + : +- Exchange (118) + : +- Project (117) + : +- Filter (116) + : +- Scan parquet (115) + +- Sort (138) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -512,181 +522,221 @@ Condition : isnotnull(s_nationkey#X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Scan parquet +(97) Sort +Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(98) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(98) Filter +(99) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(99) Exchange +(100) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(101) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 + +(102) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(103) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(104) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(105) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(106) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(107) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(109) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 + +(110) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(107) Filter +(111) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(108) Project +(112) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(109) Exchange +(113) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(114) Sort +Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(115) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(112) Project +(117) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(113) Exchange +(118) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(120) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(115) HashAggregate +(121) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(116) HashAggregate +(122) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(117) Filter +(123) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(118) Exchange +(124) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(125) Sort +Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(120) Project +(127) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(121) Exchange +(128) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(129) Sort +Input [1]: [ps_suppkey#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 + +(130) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(123) Project +(131) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(124) Exchange +(132) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Scan parquet +(133) Sort +Input [3]: [s_name#X, s_address#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(126) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(127) Project +(136) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(128) Exchange +(137) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) ShuffledHashJoin +(138) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(139) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(130) Project +(140) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(131) Exchange +(141) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(142) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(133) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index 7b8c173fc086..5e8c9ad9f92a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (137) +- == Final Plan == VeloxColumnarToRowExec (91) +- ^ RegularHashAggregateExecTransformer (89) @@ -72,42 +72,51 @@ AdaptiveSparkPlan (128) +- ^ FilterExecTransformer (71) +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (110) - : : +- Project (109) - : : +- ShuffledHashJoin Inner BuildLeft (108) - : : :- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Exchange (107) - : : +- ShuffledHashJoin LeftAnti BuildRight (106) - : : :- ShuffledHashJoin LeftSemi BuildRight (101) - : : : :- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Exchange (100) - : : : +- Scan parquet (99) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Exchange (114) - : +- Project (113) - : +- Filter (112) - : +- Scan parquet (111) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + TakeOrderedAndProject (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin Inner (122) + : :- Sort (116) + : : +- Exchange (115) + : : +- Project (114) + : : +- SortMergeJoin Inner (113) + : : :- Sort (95) + : : : +- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Sort (112) + : : +- Exchange (111) + : : +- SortMergeJoin LeftAnti (110) + : : :- SortMergeJoin LeftSemi (104) + : : : :- Sort (100) + : : : : +- Exchange (99) + : : : : +- Project (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (103) + : : : +- Exchange (102) + : : : +- Scan parquet (101) + : : +- Sort (109) + : : +- Exchange (108) + : : +- Project (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- Filter (118) + : +- Scan parquet (117) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) (1) Scan parquet @@ -501,168 +510,204 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Scan parquet +(95) Sort +Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(100) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(101) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(100) Exchange +(102) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) ShuffledHashJoin +(103) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(102) Scan parquet +(105) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(103) Filter +(106) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(104) Project +(107) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(105) Exchange +(108) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(109) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(107) Exchange +(111) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(112) Sort +Input [2]: [l_orderkey#X, l_suppkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(109) Project +(114) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(110) Exchange +(115) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(116) Sort +Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(117) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(112) Filter +(118) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(113) Project +(119) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(114) Exchange +(120) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(121) Sort +Input [1]: [o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(116) Project +(123) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(117) Exchange +(124) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(125) Sort +Input [2]: [s_name#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(126) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(127) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(120) Project +(128) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(129) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(130) Sort +Input [1]: [n_nationkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(123) Project +(132) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(124) HashAggregate +(133) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(125) Exchange +(134) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(135) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(127) TakeOrderedAndProject +(136) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(128) AdaptiveSparkPlan +(137) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index d6ec93a97fc6..50f1c1bdef30 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (52) +- == Final Plan == VeloxColumnarToRowExec (37) +- ^ SortExecTransformer (35) @@ -30,18 +30,20 @@ AdaptiveSparkPlan (50) +- ^ ProjectExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftAnti BuildRight (43) - :- Exchange (40) - : +- Filter (39) - : +- Scan parquet (38) - +- Exchange (42) - +- Scan parquet (41) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftAnti (45) + :- Sort (41) + : +- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Sort (44) + +- Exchange (43) + +- Scan parquet (42) (1) Scan parquet @@ -211,192 +213,200 @@ Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23 Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(41) Sort +Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(42) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(42) Exchange +(43) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(44) Sort +Input [1]: [o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(44) Project +(46) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(47) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(48) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(49) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(50) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(51) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (70) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ InputIteratorTransformer (60) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- VeloxAppendBatches (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ Scan parquet (53) +- == Initial Plan == - HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) -(51) Scan parquet +(53) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) ProjectExecTransformer +(55) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) FlushableHashAggregateExecTransformer +(56) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(56) VeloxAppendBatches +(58) VeloxAppendBatches Input [2]: [sum#X, count#X] Arguments: X -(57) ColumnarExchange +(59) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(60) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(59) InputAdapter +(61) InputAdapter Input [2]: [sum#X, count#X] -(60) InputIteratorTransformer +(62) InputIteratorTransformer Input [2]: [sum#X, count#X] -(61) RegularHashAggregateExecTransformer +(63) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(62) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(63) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(64) Scan parquet +(66) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(65) Filter +(67) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(66) Project +(68) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(67) HashAggregate +(69) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(68) Exchange +(70) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(69) HashAggregate +(71) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(70) AdaptiveSparkPlan +(72) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (70) +AdaptiveSparkPlan (72) +- == Final Plan == - VeloxColumnarToRowExec (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ InputIteratorTransformer (60) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- VeloxAppendBatches (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ FilterExecTransformer (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (65) + +- ^ RegularHashAggregateExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ FlushableHashAggregateExecTransformer (56) + +- ^ ProjectExecTransformer (55) + +- ^ FilterExecTransformer (54) + +- ^ Scan parquet (53) +- == Initial Plan == - HashAggregate (69) - +- Exchange (68) - +- HashAggregate (67) - +- Project (66) - +- Filter (65) - +- Scan parquet (64) \ No newline at end of file + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- Filter (67) + +- Scan parquet (66) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index 709a1700b5c7..50ad3b59c347 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (67) +- == Final Plan == VeloxColumnarToRowExec (43) +- TakeOrderedAndProjectExecTransformer (42) @@ -36,25 +36,29 @@ AdaptiveSparkPlan (63) +- ^ FilterExecTransformer (28) +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (62) - +- HashAggregate (61) - +- HashAggregate (60) - +- Project (59) - +- ShuffledHashJoin Inner BuildRight (58) - :- Exchange (53) - : +- Project (52) - : +- ShuffledHashJoin Inner BuildLeft (51) - : :- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Exchange (50) - : +- Filter (49) - : +- Scan parquet (48) - +- Exchange (57) - +- Project (56) - +- Filter (55) - +- Scan parquet (54) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Project (63) + +- SortMergeJoin Inner (62) + :- Sort (56) + : +- Exchange (55) + : +- Project (54) + : +- SortMergeJoin Inner (53) + : :- Sort (48) + : : +- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Sort (52) + : +- Exchange (51) + : +- Filter (50) + : +- Scan parquet (49) + +- Sort (61) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -258,82 +262,98 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Scan parquet +(48) Sort +Input [1]: [c_custkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(49) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(49) Filter +(50) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(50) Exchange +(51) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) ShuffledHashJoin +(52) Sort +Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(53) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(52) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(53) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(56) Sort +Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(57) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(55) Filter +(58) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(56) Project +(59) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(57) Exchange +(60) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) ShuffledHashJoin +(61) Sort +Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(62) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) Project +(63) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(60) HashAggregate +(64) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(61) HashAggregate +(65) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(62) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(63) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index a82dbf288086..2db46753e9fc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (56) +- == Final Plan == VeloxColumnarToRowExec (38) +- ^ SortExecTransformer (36) @@ -31,21 +31,23 @@ AdaptiveSparkPlan (54) +- ^ FilterExecTransformer (11) +- ^ Scan parquet (10) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- ShuffledHashJoin LeftSemi BuildRight (47) - :- Exchange (42) - : +- Project (41) - : +- Filter (40) - : +- Scan parquet (39) - +- Exchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + Sort (55) + +- Exchange (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Project (50) + +- SortMergeJoin LeftSemi (49) + :- Sort (43) + : +- Exchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- Sort (48) + +- Exchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -224,61 +226,69 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Scan parquet +(43) Sort +Input [2]: [o_orderkey#X, o_orderpriority#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(44) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(44) Filter +(45) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(45) Project +(46) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(46) Exchange +(47) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(48) Sort +Input [1]: [l_orderkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(49) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(48) Project +(50) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(51) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(52) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(54) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(55) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index f20a52b91a8f..07a5c86709f4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (156) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,45 +83,55 @@ AdaptiveSparkPlan (146) +- ^ FilterExecTransformer (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- HashAggregate (143) - +- Exchange (142) - +- HashAggregate (141) - +- Project (140) - +- ShuffledHashJoin Inner BuildRight (139) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Project (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (138) - +- Project (137) - +- Filter (136) - +- Scan parquet (135) + Sort (155) + +- Exchange (154) + +- HashAggregate (153) + +- Exchange (152) + +- HashAggregate (151) + +- Project (150) + +- SortMergeJoin Inner (149) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (110) + : : : : : +- Exchange (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Project (113) + : : : : +- Filter (112) + : : : : +- Scan parquet (111) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (148) + +- Exchange (147) + +- Project (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -572,181 +582,221 @@ Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(110) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(111) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(111) Filter +(112) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(112) Project +(113) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(113) Exchange +(114) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(115) Project +(117) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(118) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [2]: [c_nationkey#X, o_orderkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(119) Exchange +(122) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(121) Project +(125) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(122) Exchange +(126) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(125) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(127) Project +(133) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(128) Exchange +(134) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(131) Exchange +(138) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(133) Project +(141) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(134) Exchange +(142) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(137) Project +(146) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(138) Exchange +(147) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(148) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(140) Project +(150) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(141) HashAggregate +(151) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(142) Exchange +(152) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(143) HashAggregate +(153) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(144) Exchange +(154) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(155) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(146) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index 710c6f3ba189..b27398e415d3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (139) +AdaptiveSparkPlan (149) +- == Final Plan == VeloxColumnarToRowExec (101) +- ^ SortExecTransformer (99) @@ -79,43 +79,53 @@ AdaptiveSparkPlan (139) +- ShuffleQueryStage (79), Statistics(X) +- ReusedExchange (78) +- == Initial Plan == - Sort (138) - +- Exchange (137) - +- HashAggregate (136) - +- Exchange (135) - +- HashAggregate (134) - +- Project (133) - +- ShuffledHashJoin Inner BuildRight (132) - :- Exchange (128) - : +- Project (127) - : +- ShuffledHashJoin Inner BuildRight (126) - : :- Exchange (122) - : : +- Project (121) - : : +- ShuffledHashJoin Inner BuildRight (120) - : : :- Exchange (116) - : : : +- Project (115) - : : : +- ShuffledHashJoin Inner BuildRight (114) - : : : :- Exchange (110) - : : : : +- Project (109) - : : : : +- ShuffledHashJoin Inner BuildLeft (108) - : : : : :- Exchange (104) - : : : : : +- Filter (103) - : : : : : +- Scan parquet (102) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Exchange (113) - : : : +- Filter (112) - : : : +- Scan parquet (111) - : : +- Exchange (119) - : : +- Filter (118) - : : +- Scan parquet (117) - : +- Exchange (125) - : +- Filter (124) - : +- Scan parquet (123) - +- Exchange (131) - +- Filter (130) - +- Scan parquet (129) + Sort (148) + +- Exchange (147) + +- HashAggregate (146) + +- Exchange (145) + +- HashAggregate (144) + +- Project (143) + +- SortMergeJoin Inner (142) + :- Sort (137) + : +- Exchange (136) + : +- Project (135) + : +- SortMergeJoin Inner (134) + : :- Sort (129) + : : +- Exchange (128) + : : +- Project (127) + : : +- SortMergeJoin Inner (126) + : : :- Sort (121) + : : : +- Exchange (120) + : : : +- Project (119) + : : : +- SortMergeJoin Inner (118) + : : : :- Sort (113) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- SortMergeJoin Inner (110) + : : : : :- Sort (105) + : : : : : +- Exchange (104) + : : : : : +- Filter (103) + : : : : : +- Scan parquet (102) + : : : : +- Sort (109) + : : : : +- Exchange (108) + : : : : +- Filter (107) + : : : : +- Scan parquet (106) + : : : +- Sort (117) + : : : +- Exchange (116) + : : : +- Filter (115) + : : : +- Scan parquet (114) + : : +- Sort (125) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Sort (133) + : +- Exchange (132) + : +- Filter (131) + : +- Scan parquet (130) + +- Sort (141) + +- Exchange (140) + +- Filter (139) + +- Scan parquet (138) (1) Scan parquet @@ -542,173 +552,213 @@ Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(105) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(106) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(107) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(108) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(109) Sort +Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(110) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(109) Project +(111) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(110) Exchange +(112) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(113) Sort +Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(114) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(112) Filter +(115) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(113) Exchange +(116) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(117) Sort +Input [2]: [o_orderkey#X, o_custkey#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(115) Project +(119) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(120) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(121) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(122) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(118) Filter +(123) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(119) Exchange +(124) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(125) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(126) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(121) Project +(127) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(122) Exchange +(128) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(129) Sort +Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(130) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(124) Filter +(131) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(125) Exchange +(132) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(133) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(134) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(135) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(128) Exchange +(136) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(137) Sort +Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(138) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(130) Filter +(139) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(131) Exchange +(140) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(141) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(133) Project +(143) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(134) HashAggregate +(144) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(135) Exchange +(145) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) HashAggregate +(146) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(137) Exchange +(147) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(148) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(139) AdaptiveSparkPlan +(149) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index 953a3a8f0a7c..fa2a2789f4d3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (193) +AdaptiveSparkPlan (207) +- == Final Plan == VeloxColumnarToRowExec (141) +- ^ SortExecTransformer (139) @@ -110,57 +110,71 @@ AdaptiveSparkPlan (193) +- ^ FilterExecTransformer (113) +- ^ Scan parquet (112) +- == Initial Plan == - Sort (192) - +- Exchange (191) - +- HashAggregate (190) - +- Exchange (189) - +- HashAggregate (188) - +- Project (187) - +- ShuffledHashJoin Inner BuildRight (186) - :- Exchange (181) - : +- Project (180) - : +- ShuffledHashJoin Inner BuildRight (179) - : :- Exchange (175) - : : +- Project (174) - : : +- ShuffledHashJoin Inner BuildRight (173) - : : :- Exchange (169) - : : : +- Project (168) - : : : +- ShuffledHashJoin Inner BuildRight (167) - : : : :- Exchange (163) - : : : : +- Project (162) - : : : : +- ShuffledHashJoin Inner BuildRight (161) - : : : : :- Exchange (157) - : : : : : +- Project (156) - : : : : : +- ShuffledHashJoin Inner BuildRight (155) - : : : : : :- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (149) - : : : : : : :- Exchange (145) - : : : : : : : +- Project (144) - : : : : : : : +- Filter (143) - : : : : : : : +- Scan parquet (142) - : : : : : : +- Exchange (148) - : : : : : : +- Filter (147) - : : : : : : +- Scan parquet (146) - : : : : : +- Exchange (154) - : : : : : +- Filter (153) - : : : : : +- Scan parquet (152) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Exchange (166) - : : : +- Filter (165) - : : : +- Scan parquet (164) - : : +- Exchange (172) - : : +- Filter (171) - : : +- Scan parquet (170) - : +- Exchange (178) - : +- Filter (177) - : +- Scan parquet (176) - +- Exchange (185) - +- Project (184) - +- Filter (183) - +- Scan parquet (182) + Sort (206) + +- Exchange (205) + +- HashAggregate (204) + +- Exchange (203) + +- HashAggregate (202) + +- Project (201) + +- SortMergeJoin Inner (200) + :- Sort (194) + : +- Exchange (193) + : +- Project (192) + : +- SortMergeJoin Inner (191) + : :- Sort (186) + : : +- Exchange (185) + : : +- Project (184) + : : +- SortMergeJoin Inner (183) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Project (176) + : : : +- SortMergeJoin Inner (175) + : : : :- Sort (170) + : : : : +- Exchange (169) + : : : : +- Project (168) + : : : : +- SortMergeJoin Inner (167) + : : : : :- Sort (162) + : : : : : +- Exchange (161) + : : : : : +- Project (160) + : : : : : +- SortMergeJoin Inner (159) + : : : : : :- Sort (154) + : : : : : : +- Exchange (153) + : : : : : : +- Project (152) + : : : : : : +- SortMergeJoin Inner (151) + : : : : : : :- Sort (146) + : : : : : : : +- Exchange (145) + : : : : : : : +- Project (144) + : : : : : : : +- Filter (143) + : : : : : : : +- Scan parquet (142) + : : : : : : +- Sort (150) + : : : : : : +- Exchange (149) + : : : : : : +- Filter (148) + : : : : : : +- Scan parquet (147) + : : : : : +- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (166) + : : : : +- Exchange (165) + : : : : +- Filter (164) + : : : : +- Scan parquet (163) + : : : +- Sort (174) + : : : +- Exchange (173) + : : : +- Filter (172) + : : : +- Scan parquet (171) + : : +- Sort (182) + : : +- Exchange (181) + : : +- Filter (180) + : : +- Scan parquet (179) + : +- Sort (190) + : +- Exchange (189) + : +- Filter (188) + : +- Scan parquet (187) + +- Sort (199) + +- Exchange (198) + +- Project (197) + +- Filter (196) + +- Scan parquet (195) (1) Scan parquet @@ -757,235 +771,291 @@ Input [2]: [p_partkey#X, p_type#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Scan parquet +(146) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(147) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(147) Filter +(148) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(148) Exchange +(149) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(149) ShuffledHashJoin +(150) Sort +Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(151) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(150) Project +(152) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(153) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Scan parquet +(154) Sort +Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(155) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(153) Filter +(156) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(154) Exchange +(157) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(155) ShuffledHashJoin +(158) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(159) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(156) Project +(160) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(157) Exchange +(161) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(158) Scan parquet +(162) Sort +Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(163) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(159) Filter +(164) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(160) Exchange +(165) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(161) ShuffledHashJoin +(166) Sort +Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(167) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(162) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(163) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Scan parquet +(170) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 + +(171) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(165) Filter +(172) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(166) Exchange +(173) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(167) ShuffledHashJoin +(174) Sort +Input [2]: [c_custkey#X, c_nationkey#X] +Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 + +(175) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(168) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(169) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) Scan parquet +(178) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 + +(179) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(171) Filter +(180) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(172) Exchange +(181) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(173) ShuffledHashJoin +(182) Sort +Input [2]: [n_nationkey#X, n_regionkey#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(183) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(174) Project +(184) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(175) Exchange +(185) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Scan parquet +(186) Sort +Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(187) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(177) Filter +(188) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(178) Exchange +(189) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(179) ShuffledHashJoin +(190) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(191) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(180) Project +(192) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(181) Exchange +(193) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(182) Scan parquet +(194) Sort +Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 + +(195) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(183) Filter +(196) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(184) Project +(197) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(185) Exchange +(198) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(186) ShuffledHashJoin +(199) Sort +Input [1]: [r_regionkey#X] +Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 + +(200) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(187) Project +(201) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(188) HashAggregate +(202) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(189) Exchange +(203) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(190) HashAggregate +(204) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(191) Exchange +(205) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(206) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(193) AdaptiveSparkPlan +(207) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index 492ff1aeadd0..3000cbae7a6d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -1,5 +1,5 @@ == Physical Plan == -AdaptiveSparkPlan (145) +AdaptiveSparkPlan (155) +- == Final Plan == VeloxColumnarToRowExec (106) +- ^ SortExecTransformer (104) @@ -83,44 +83,54 @@ AdaptiveSparkPlan (145) +- ^ FilterExecTransformer (79) +- ^ Scan parquet (78) +- == Initial Plan == - Sort (144) - +- Exchange (143) - +- HashAggregate (142) - +- Exchange (141) - +- HashAggregate (140) - +- Project (139) - +- ShuffledHashJoin Inner BuildRight (138) - :- Exchange (134) - : +- Project (133) - : +- ShuffledHashJoin Inner BuildRight (132) - : :- Exchange (128) - : : +- Project (127) - : : +- ShuffledHashJoin Inner BuildRight (126) - : : :- Exchange (122) - : : : +- Project (121) - : : : +- ShuffledHashJoin Inner BuildRight (120) - : : : :- Exchange (116) - : : : : +- Project (115) - : : : : +- ShuffledHashJoin Inner BuildLeft (114) - : : : : :- Exchange (110) - : : : : : +- Project (109) - : : : : : +- Filter (108) - : : : : : +- Scan parquet (107) - : : : : +- Exchange (113) - : : : : +- Filter (112) - : : : : +- Scan parquet (111) - : : : +- Exchange (119) - : : : +- Filter (118) - : : : +- Scan parquet (117) - : : +- Exchange (125) - : : +- Filter (124) - : : +- Scan parquet (123) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Exchange (137) - +- Filter (136) - +- Scan parquet (135) + Sort (154) + +- Exchange (153) + +- HashAggregate (152) + +- Exchange (151) + +- HashAggregate (150) + +- Project (149) + +- SortMergeJoin Inner (148) + :- Sort (143) + : +- Exchange (142) + : +- Project (141) + : +- SortMergeJoin Inner (140) + : :- Sort (135) + : : +- Exchange (134) + : : +- Project (133) + : : +- SortMergeJoin Inner (132) + : : :- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- SortMergeJoin Inner (124) + : : : :- Sort (119) + : : : : +- Exchange (118) + : : : : +- Project (117) + : : : : +- SortMergeJoin Inner (116) + : : : : :- Sort (111) + : : : : : +- Exchange (110) + : : : : : +- Project (109) + : : : : : +- Filter (108) + : : : : : +- Scan parquet (107) + : : : : +- Sort (115) + : : : : +- Exchange (114) + : : : : +- Filter (113) + : : : : +- Scan parquet (112) + : : : +- Sort (123) + : : : +- Exchange (122) + : : : +- Filter (121) + : : : +- Scan parquet (120) + : : +- Sort (131) + : : +- Exchange (130) + : : +- Filter (129) + : : +- Scan parquet (128) + : +- Sort (139) + : +- Exchange (138) + : +- Filter (137) + : +- Scan parquet (136) + +- Sort (147) + +- Exchange (146) + +- Filter (145) + +- Scan parquet (144) (1) Scan parquet @@ -575,173 +585,213 @@ Input [2]: [p_partkey#X, p_name#X] Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(111) Sort +Input [1]: [p_partkey#X] +Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 + +(112) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(112) Filter +(113) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(113) Exchange +(114) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(115) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 + +(116) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(115) Project +(117) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(118) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(119) Sort +Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 + +(120) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(118) Filter +(121) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(119) Exchange +(122) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(123) Sort +Input [2]: [s_suppkey#X, s_nationkey#X] +Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 + +(124) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(121) Project +(125) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(122) Exchange +(126) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(127) Sort +Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 + +(128) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(124) Filter +(129) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(125) Exchange +(130) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(131) Sort +Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 + +(132) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(127) Project +(133) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(128) Exchange +(134) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Scan parquet +(135) Sort +Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 + +(136) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(130) Filter +(137) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(131) Exchange +(138) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) ShuffledHashJoin +(139) Sort +Input [2]: [o_orderkey#X, o_orderdate#X] +Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 + +(140) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(133) Project +(141) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(134) Exchange +(142) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) Scan parquet +(143) Sort +Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 + +(144) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(136) Filter +(145) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(137) Exchange +(146) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) ShuffledHashJoin +(147) Sort +Input [2]: [n_nationkey#X, n_name#X] +Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 + +(148) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(139) Project +(149) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(140) HashAggregate +(150) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(141) Exchange +(151) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) HashAggregate +(152) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(143) Exchange +(153) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(154) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(145) AdaptiveSparkPlan +(155) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index 27d191b9ee05..2b40ac54b2c6 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -23,6 +23,7 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.execution.{ColumnarShuffleExchangeExec, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEShuffleReadExec} import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec +import org.apache.spark.sql.execution.joins.SortMergeJoinExec class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPlanHelper { protected val rootPath: String = getClass.getResource("/").getPath @@ -240,4 +241,26 @@ class FallbackSuite extends VeloxWholeStageTransformerSuite with AdaptiveSparkPl } } } + + test("fallback with smj") { + val sql = "SELECT /*+ SHUFFLE_MERGE(tmp1) */ * FROM tmp1 join tmp2 on tmp1.c1 = tmp2.c1" + withSQLConf( + GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key -> "true", + GlutenConfig.COLUMNAR_SHUFFLED_HASH_JOIN_ENABLED.key -> "false") { + runQueryAndCompare(sql) { + df => + val plan = df.queryExecution.executedPlan + assert(collect(plan) { case smj: SortMergeJoinExec => smj }.size == 1) + } + } + withSQLConf( + GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key -> "false", + GlutenConfig.COLUMNAR_SORTMERGEJOIN_ENABLED.key -> "false") { + runQueryAndCompare(sql) { + df => + val plan = df.queryExecution.executedPlan + assert(collect(plan) { case smj: SortMergeJoinExec => smj }.size == 1) + } + } + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 2860e3ced072..0f397c69263c 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -22,7 +22,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.events.GlutenBuildInfoEvent import org.apache.gluten.exception.GlutenException import org.apache.gluten.expression.ExpressionMappings -import org.apache.gluten.extension.{ColumnarOverrides, OthersExtensionOverrides, QueryStagePrepOverrides, StrategyOverrides} +import org.apache.gluten.extension.{ColumnarOverrides, OthersExtensionOverrides, QueryStagePrepOverrides} import org.apache.gluten.test.TestStats import org.apache.gluten.utils.TaskListener @@ -312,7 +312,6 @@ private[gluten] object GlutenPlugin { val DEFAULT_INJECTORS: List[GlutenSparkExtensionsInjector] = List( QueryStagePrepOverrides, ColumnarOverrides, - StrategyOverrides, OthersExtensionOverrides ) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index 50292839b684..d159486373ac 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -86,7 +86,6 @@ trait BackendSettingsApi { * the result columns from the shuffle. */ def supportShuffleWithProject(outputPartitioning: Partitioning, child: SparkPlan): Boolean = false - def utilizeShuffledHashJoinHint(): Boolean = false def excludeScanExecFromCollapsedStage(): Boolean = false def rescaleDecimalArithmetic: Boolean = false diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala similarity index 53% rename from shims/spark32/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala rename to gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala index 5849dd600b7d..2c0ad1b0a59a 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala @@ -14,27 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution +package org.apache.gluten.execution -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys -import org.apache.spark.sql.catalyst.plans.JoinType -import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint, LogicalPlan} +import org.apache.gluten.extension.columnar.rewrite.RewrittenNodeWall -// https://issues.apache.org/jira/browse/SPARK-36745 -object JoinSelectionShim { - object ExtractEquiJoinKeysShim { - type ReturnType = - ( - JoinType, - Seq[Expression], - Seq[Expression], - Option[Expression], - LogicalPlan, - LogicalPlan, - JoinHint) - def unapply(join: Join): Option[ReturnType] = { - ExtractEquiJoinKeys.unapply(join) - } +import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan} + +object SortUtils { + def dropPartialSort(plan: SparkPlan): SparkPlan = plan match { + case RewrittenNodeWall(p) => RewrittenNodeWall(dropPartialSort(p)) + case sort: SortExec if !sort.global => sort.child + // from pre/post project-pulling + case ProjectExec(_, SortExec(_, false, ProjectExec(_, p), _)) + if plan.outputSet == p.outputSet => + p + case _ => plan } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala deleted file mode 100644 index f2f786259393..000000000000 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/StrategyOverrides.scala +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.extension - -import org.apache.gluten.{GlutenConfig, GlutenSparkExtensionsInjector} -import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.extension.columnar.TRANSFORM_UNSUPPORTED -import org.apache.gluten.extension.columnar.TransformHints.TAG -import org.apache.gluten.utils.LogicalPlanSelector - -import org.apache.spark.sql.{SparkSession, SparkSessionExtensions, Strategy} -import org.apache.spark.sql.catalyst.SQLConfHelper -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, JoinSelectionHelper} -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.execution.{joins, JoinSelectionShim, SparkPlan} -import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec, LogicalQueryStage} -import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec - -object StrategyOverrides extends GlutenSparkExtensionsInjector { - override def inject(extensions: SparkSessionExtensions): Unit = { - extensions.injectPlannerStrategy(JoinSelectionOverrides) - } -} - -case class JoinSelectionOverrides(session: SparkSession) - extends Strategy - with JoinSelectionHelper - with SQLConfHelper { - - private def isBroadcastStage(plan: LogicalPlan): Boolean = plan match { - case LogicalQueryStage(_, _: BroadcastQueryStageExec) => true - case _ => false - } - - def extractEqualJoinKeyCondition( - joinType: JoinType, - leftKeys: Seq[Expression], - rightKeys: Seq[Expression], - condition: Option[Expression], - left: LogicalPlan, - right: LogicalPlan, - hint: JoinHint, - forceShuffledHashJoin: Boolean): Seq[SparkPlan] = { - if (isBroadcastStage(left) || isBroadcastStage(right)) { - val buildSide = if (isBroadcastStage(left)) BuildLeft else BuildRight - Seq( - BroadcastHashJoinExec( - leftKeys, - rightKeys, - joinType, - buildSide, - condition, - planLater(left), - planLater(right))) - } else { - // Generate BHJ here, avoid to do match in `JoinSelection` again. - val isHintEmpty = hint.leftHint.isEmpty && hint.rightHint.isEmpty - val buildSide = getBroadcastBuildSide(left, right, joinType, hint, !isHintEmpty, conf) - if (buildSide.isDefined) { - return Seq( - joins.BroadcastHashJoinExec( - leftKeys, - rightKeys, - joinType, - buildSide.get, - condition, - planLater(left), - planLater(right))) - } - - if ( - forceShuffledHashJoin && - !BackendsApiManager.getSparkPlanExecApiInstance.joinFallback( - joinType, - left.outputSet, - right.outputSet, - condition) && - !left.getTagValue(TAG).isDefined && - !right.getTagValue(TAG).isDefined - ) { - // Force use of ShuffledHashJoin in preference to SortMergeJoin. With no respect to - // conf setting "spark.sql.join.preferSortMergeJoin". - val (leftBuildable, rightBuildable) = - if (BackendsApiManager.getSettings.utilizeShuffledHashJoinHint()) { - // Currently, ClickHouse backend can not support AQE, so it needs to use join hint - // to decide the build side, after supporting AQE, will remove this. - val leftHintEnabled = hintToShuffleHashJoinLeft(hint) - val rightHintEnabled = hintToShuffleHashJoinRight(hint) - val leftHintMergeEnabled = hint.leftHint.exists(_.strategy.contains(SHUFFLE_MERGE)) - val rightHintMergeEnabled = hint.rightHint.exists(_.strategy.contains(SHUFFLE_MERGE)) - if (leftHintEnabled || rightHintEnabled) { - (leftHintEnabled, rightHintEnabled) - } else if (leftHintMergeEnabled || rightHintMergeEnabled) { - // hack: when set SHUFFLE_MERGE hint, it means that - // it don't use this side as the build side - (!leftHintMergeEnabled, !rightHintMergeEnabled) - } else { - ( - BackendsApiManager.getSettings.supportHashBuildJoinTypeOnLeft(joinType), - BackendsApiManager.getSettings.supportHashBuildJoinTypeOnRight(joinType)) - } - } else { - (canBuildShuffledHashJoinLeft(joinType), canBuildShuffledHashJoinRight(joinType)) - } - - if (!leftBuildable && !rightBuildable) { - return Nil - } - val buildSide = if (!leftBuildable) { - BuildRight - } else if (!rightBuildable) { - BuildLeft - } else { - getSmallerSide(left, right) - } - - return Option(buildSide) - .map { - buildSide => - Seq( - joins.ShuffledHashJoinExec( - leftKeys, - rightKeys, - joinType, - buildSide, - condition, - planLater(left), - planLater(right))) - } - .getOrElse(Nil) - } - Nil - } - } - - def existsMultiJoins(plan: LogicalPlan, count: Int = 0): Boolean = { - plan match { - case plan: Join => - if ((count + 1) >= GlutenConfig.getConf.logicalJoinOptimizationThrottle) return true - plan.children.exists(existsMultiJoins(_, count + 1)) - case plan: Project => - if ((count + 1) >= GlutenConfig.getConf.logicalJoinOptimizationThrottle) return true - plan.children.exists(existsMultiJoins(_, count + 1)) - case other => false - } - } - - def tagNotTransformable(plan: LogicalPlan, reason: String): LogicalPlan = { - plan.setTagValue(TAG, TRANSFORM_UNSUPPORTED(Some(reason))) - plan - } - - def tagNotTransformableRecursive(plan: LogicalPlan, reason: String): LogicalPlan = { - tagNotTransformable( - plan.withNewChildren(plan.children.map(tagNotTransformableRecursive(_, reason))), - reason) - } - - def existLeftOuterJoin(plan: LogicalPlan): Boolean = { - plan.collect { - case join: Join if join.joinType.sql.equals("LEFT OUTER") => - return true - }.size > 0 - } - - override def apply(plan: LogicalPlan): Seq[SparkPlan] = - LogicalPlanSelector.maybeNil(session, plan) { - // Ignore forceShuffledHashJoin if exist multi continuous joins - if ( - GlutenConfig.getConf.enableLogicalJoinOptimize && - existsMultiJoins(plan) && existLeftOuterJoin(plan) - ) { - tagNotTransformableRecursive(plan, "exist multi continuous joins") - } - plan match { - // If the build side of BHJ is already decided by AQE, we need to keep the build side. - case JoinSelectionShim.ExtractEquiJoinKeysShim( - joinType, - leftKeys, - rightKeys, - condition, - left, - right, - hint) => - extractEqualJoinKeyCondition( - joinType, - leftKeys, - rightKeys, - condition, - left, - right, - hint, - GlutenConfig.getConf.forceShuffledHashJoin) - case _ => Nil - } - } -} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 11db0bc1faf1..1f6f840b5552 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -27,6 +27,8 @@ import org.apache.gluten.utils.{LogLevelUtil, PlanUtil} import org.apache.spark.api.python.EvalPythonExecTransformer import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, InputFileBlockLength, InputFileBlockStart, InputFileName, NamedExpression} +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} +import org.apache.spark.sql.catalyst.plans.logical.Join import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} import org.apache.spark.sql.execution.datasources.WriteFilesExec @@ -136,7 +138,7 @@ case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { plan.leftKeys, plan.rightKeys, plan.joinType, - TransformHints.getShuffleHashJoinBuildSide(plan), + OffloadJoin.getBuildSide(plan), plan.condition, left, right, @@ -187,6 +189,31 @@ case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { } } +object OffloadJoin { + + def getBuildSide(shj: ShuffledHashJoinExec): BuildSide = { + val leftBuildable = + BackendsApiManager.getSettings.supportHashBuildJoinTypeOnLeft(shj.joinType) + val rightBuildable = + BackendsApiManager.getSettings.supportHashBuildJoinTypeOnRight(shj.joinType) + if (!leftBuildable) { + BuildRight + } else if (!rightBuildable) { + BuildLeft + } else { + shj.logicalLink match { + case Some(join: Join) => + val leftSize = join.left.stats.sizeInBytes + val rightSize = join.right.stats.sizeInBytes + if (rightSize <= leftSize) BuildRight else BuildLeft + // Only the ShuffledHashJoinExec generated directly in some spark tests is not link + // logical plan, such as OuterJoinSuite. + case _ => shj.buildSide + } + } + } +} + case class OffloadProject() extends OffloadSingleNode with LogLevelUtil { private def containsInputFileRelatedExpr(expr: Expression): Boolean = { expr match { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala index aa7aab759ef8..7fb451057a2e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala @@ -29,8 +29,6 @@ import org.apache.spark.api.python.EvalPythonExecTransformer import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} -import org.apache.spark.sql.catalyst.plans.logical.Join import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.spark.sql.execution._ @@ -154,33 +152,6 @@ object TransformHints { tag(plan, newTag) } } - - def getShuffleHashJoinBuildSide(shj: ShuffledHashJoinExec): BuildSide = { - if (BackendsApiManager.getSettings.utilizeShuffledHashJoinHint()) { - shj.buildSide - } else { - val leftBuildable = BackendsApiManager.getSettings - .supportHashBuildJoinTypeOnLeft(shj.joinType) - val rightBuildable = BackendsApiManager.getSettings - .supportHashBuildJoinTypeOnRight(shj.joinType) - - if (!leftBuildable) { - BuildRight - } else if (!rightBuildable) { - BuildLeft - } else { - shj.logicalLink match { - case Some(join: Join) => - val leftSize = join.left.stats.sizeInBytes - val rightSize = join.right.stats.sizeInBytes - if (rightSize <= leftSize) BuildRight else BuildLeft - // Only the ShuffledHashJoinExec generated directly in some spark tests is not link - // logical plan, such as OuterJoinSuite. - case _ => shj.buildSide - } - } - } - } } case class FallbackOnANSIMode(session: SparkSession) extends Rule[SparkPlan] { @@ -205,6 +176,9 @@ case class FallbackMultiCodegens(session: SparkSession) extends Rule[SparkPlan] case plan: ShuffledHashJoinExec => if ((count + 1) >= optimizeLevel) return true plan.children.exists(existsMultiCodegens(_, count + 1)) + case plan: SortMergeJoinExec if GlutenConfig.getConf.forceShuffledHashJoin => + if ((count + 1) >= optimizeLevel) return true + plan.children.exists(existsMultiCodegens(_, count + 1)) case other => false } @@ -415,7 +389,7 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { plan.leftKeys, plan.rightKeys, plan.joinType, - TransformHints.getShuffleHashJoinBuildSide(plan), + OffloadJoin.getBuildSide(plan), plan.condition, plan.left, plan.right, diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala new file mode 100644 index 000000000000..e038f5af0a07 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.rewrite + +import org.apache.gluten.GlutenConfig +import org.apache.gluten.execution.SortUtils + +import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide, JoinSelectionHelper} +import org.apache.spark.sql.catalyst.plans.JoinType +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec} + +/** + * If force ShuffledHashJoin, convert [[SortMergeJoinExec]] to [[ShuffledHashJoinExec]]. There is no + * need to select a smaller table as buildSide here, it will be reselected when offloading. + */ +object RewriteJoin extends RewriteSingleNode with JoinSelectionHelper { + + private def getBuildSide(joinType: JoinType): Option[BuildSide] = { + val leftBuildable = canBuildShuffledHashJoinLeft(joinType) + val rightBuildable = canBuildShuffledHashJoinRight(joinType) + if (rightBuildable) { + Some(BuildRight) + } else if (leftBuildable) { + Some(BuildLeft) + } else { + None + } + } + + override def rewrite(plan: SparkPlan): SparkPlan = plan match { + case smj: SortMergeJoinExec if GlutenConfig.getConf.forceShuffledHashJoin => + getBuildSide(smj.joinType) match { + case Some(buildSide) => + ShuffledHashJoinExec( + smj.leftKeys, + smj.rightKeys, + smj.joinType, + buildSide, + smj.condition, + SortUtils.dropPartialSort(smj.left), + SortUtils.dropPartialSort(smj.right), + smj.isSkewJoin + ) + case _ => plan + } + case _ => plan + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala index 01f2e29fe62d..551cfd599abd 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSingleNode.scala @@ -35,6 +35,6 @@ trait RewriteSingleNode { object RewriteSingleNode { def allRules(): Seq[RewriteSingleNode] = { - Seq(RewriteIn, RewriteMultiChildrenCount, PullOutPreProject, PullOutPostProject) + Seq(RewriteIn, RewriteMultiChildrenCount, RewriteJoin, PullOutPreProject, PullOutPostProject) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala index 8706e5618f6b..34fe34f3f3fa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala @@ -67,12 +67,11 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] } } - private def getTransformHintBack( - origin: SparkPlan, - rewrittenPlan: SparkPlan): Option[TransformHint] = { - // The rewritten plan may contain more nodes than origin, here use the node name to get it back + private def getTransformHintBack(rewrittenPlan: SparkPlan): Option[TransformHint] = { + // The rewritten plan may contain more nodes than origin, for now it should only be + // `ProjectExec`. val target = rewrittenPlan.collect { - case p if p.nodeName == origin.nodeName => p + case p if !p.isInstanceOf[ProjectExec] && !p.isInstanceOf[RewrittenNodeWall] => p } assert(target.size == 1) TransformHints.getHintOption(target.head) @@ -113,7 +112,7 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] origin } else { addHint.apply(rewrittenPlan) - val hint = getTransformHintBack(origin, rewrittenPlan) + val hint = getTransformHintBack(rewrittenPlan) if (hint.isDefined) { // If the rewritten plan is still not transformable, return the original plan. TransformHints.tag(origin, hint.get) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5df53953e4cc..a17f72de3121 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -331,7 +331,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Shouldn't change broadcast join buildSide if user clearly specified") .exclude("Shouldn't bias towards build right if user didn't specify") .exclude("SPARK-23192: broadcast hint should be retained after using the cached data") - .exclude("broadcast hint isn't propagated after a join") .exclude("broadcast join where streamed side's output partitioning is HashPartitioning") .exclude("broadcast join where streamed side's output partitioning is PartitioningCollection") enableSuite[GlutenSQLQuerySuite] diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index c9ccc1afc75d..f418ec06645c 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -22,7 +22,6 @@ import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} import org.apache.spark.sql.catalyst.optimizer._ -import org.apache.spark.sql.execution.exchange.EnsureRequirements import org.apache.spark.sql.functions.broadcast import org.apache.spark.sql.internal.SQLConf @@ -41,8 +40,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon * Create a new [[SparkSession]] running in local-cluster mode with unsafe and codegen enabled. */ - private val EnsureRequirements = new EnsureRequirements() - private val isVeloxBackend = BackendTestUtils.isVeloxBackendLoaded() // BroadcastHashJoinExecTransformer is not case class, can't call toString method, @@ -235,22 +232,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon } } - testGluten("broadcast hint isn't propagated after a join") { - withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - val df1 = Seq((1, "4"), (2, "2")).toDF("key", "value") - val df2 = Seq((1, "1"), (2, "2")).toDF("key", "value") - val df3 = df1.join(broadcast(df2), Seq("key"), "inner").drop(df2("key")) - - val df4 = Seq((1, "5"), (2, "5")).toDF("key", "value") - val df5 = df4.join(df3, Seq("key"), "inner") - - val plan = EnsureRequirements.apply(df5.queryExecution.sparkPlan) - - assert(plan.collect { case p: BroadcastHashJoinExec => p }.size === 1) - assert(plan.collect { case p: ShuffledHashJoinExec => p }.size === 1) - } - } - private def assertJoinBuildSide(sqlStr: String, joinMethod: String, buildSide: BuildSide): Any = { val executedPlan = stripAQEPlan(sql(sqlStr).queryExecution.executedPlan) executedPlan match { diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala index 00b4bf5821f1..4df9c63b3ef6 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.extension.{ColumnarOverrideRules, JoinSelectionOverrides} +import org.apache.gluten.extension.ColumnarOverrideRules import org.apache.spark.SparkConf import org.apache.spark.sql._ @@ -31,7 +31,6 @@ class GlutenSessionExtensionSuite extends GlutenSQLTestsTrait { testGluten("test gluten extensions") { assert(spark.sessionState.columnarRules.contains(ColumnarOverrideRules(spark))) - assert(spark.sessionState.planner.strategies.contains(JoinSelectionOverrides(spark))) assert(spark.sessionState.planner.strategies.contains(MySparkStrategy(spark))) assert(spark.sessionState.analyzer.extendedResolutionRules.contains(MyRule(spark))) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 8418cba237e3..ae3e7c7b8e9d 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -865,7 +865,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Shouldn't change broadcast join buildSide if user clearly specified") .exclude("Shouldn't bias towards build right if user didn't specify") .exclude("SPARK-23192: broadcast hint should be retained after using the cached data") - .exclude("broadcast hint isn't propagated after a join") .exclude("broadcast join where streamed side's output partitioning is HashPartitioning") enableSuite[GlutenExistenceJoinSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala index 026e2dde0055..92e6fee97ea9 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala @@ -85,11 +85,11 @@ class GlutenReplaceHashWithSortAggSuite withTempView("t1", "t2") { spark.range(100).selectExpr("id as key").createOrReplaceTempView("t1") spark.range(50).selectExpr("id as key").createOrReplaceTempView("t2") - Seq("COUNT", "COLLECT_LIST").foreach { - aggExpr => + Seq(("COUNT", 0, 1, 2, 0), ("COLLECT_LIST", 2, 0, 2, 0)).foreach { + aggExprInfo => val query = s""" - |SELECT key, $aggExpr(key) + |SELECT key, ${aggExprInfo._1}(key) |FROM |( | SELECT /*+ SHUFFLE_MERGE(t1) */ t1.key AS key @@ -102,7 +102,7 @@ class GlutenReplaceHashWithSortAggSuite if (BackendsApiManager.getSettings.mergeTwoPhasesHashBaseAggregateIfNeed()) { checkAggs(query, 1, 0, 1, 0) } else { - checkAggs(query, 2, 0, 2, 0) + checkAggs(query, aggExprInfo._2, aggExprInfo._3, aggExprInfo._4, aggExprInfo._5) } } } diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala index 00b4bf5821f1..4df9c63b3ef6 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.extension.{ColumnarOverrideRules, JoinSelectionOverrides} +import org.apache.gluten.extension.ColumnarOverrideRules import org.apache.spark.SparkConf import org.apache.spark.sql._ @@ -31,7 +31,6 @@ class GlutenSessionExtensionSuite extends GlutenSQLTestsTrait { testGluten("test gluten extensions") { assert(spark.sessionState.columnarRules.contains(ColumnarOverrideRules(spark))) - assert(spark.sessionState.planner.strategies.contains(JoinSelectionOverrides(spark))) assert(spark.sessionState.planner.strategies.contains(MySparkStrategy(spark))) assert(spark.sessionState.analyzer.extendedResolutionRules.contains(MyRule(spark))) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 91127c4ba9bb..0da19922ffda 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -857,7 +857,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Shouldn't change broadcast join buildSide if user clearly specified") .exclude("Shouldn't bias towards build right if user didn't specify") .exclude("SPARK-23192: broadcast hint should be retained after using the cached data") - .exclude("broadcast hint isn't propagated after a join") .exclude("broadcast join where streamed side's output partitioning is HashPartitioning") .exclude("broadcast join where streamed side's output partitioning is PartitioningCollection") @@ -1119,9 +1118,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenJoinSuite] // exclude as it check spark plan .exclude("SPARK-36794: Ignore duplicated key when building relation for semi/anti hash join") - // exclude as it check for SMJ node - .exclude( - "SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") enableSuite[GlutenMathFunctionsSuite] enableSuite[GlutenMetadataCacheSuite] .exclude("SPARK-16336,SPARK-27961 Suggest fixing FileNotFoundException") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala index 4ac8bd3ea8bf..8a5a5923f729 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec - class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait { override def testNameBlackList: Seq[String] = Seq( @@ -55,14 +52,4 @@ class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait { |""".stripMargin checkAnswer(spark.sql(sql), Seq(Row(0, 1), Row(1, 2), Row(2, 3))) } - - testGluten( - "SPARK-43113: Full outer join with duplicate stream-side" + - " references in condition (SHJ)") { - def check(plan: SparkPlan): Unit = { - assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1) - } - - dupStreamSideColTest("MERGE", check) - } } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala index 8d795bbffea0..332c21418a9b 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala @@ -84,11 +84,11 @@ class GlutenReplaceHashWithSortAggSuite withTempView("t1", "t2") { spark.range(100).selectExpr("id as key").createOrReplaceTempView("t1") spark.range(50).selectExpr("id as key").createOrReplaceTempView("t2") - Seq("COUNT", "COLLECT_LIST").foreach { - aggExpr => + Seq(("COUNT", 0, 1, 2, 0), ("COLLECT_LIST", 2, 0, 2, 0)).foreach { + aggExprInfo => val query = s""" - |SELECT key, $aggExpr(key) + |SELECT key, ${aggExprInfo._1}(key) |FROM |( | SELECT /*+ SHUFFLE_MERGE(t1) */ t1.key AS key @@ -98,7 +98,7 @@ class GlutenReplaceHashWithSortAggSuite |) |GROUP BY key """.stripMargin - checkAggs(query, 2, 0, 2, 0) + checkAggs(query, aggExprInfo._2, aggExprInfo._3, aggExprInfo._4, aggExprInfo._5) } } } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala index 00b4bf5821f1..4df9c63b3ef6 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.extension.{ColumnarOverrideRules, JoinSelectionOverrides} +import org.apache.gluten.extension.ColumnarOverrideRules import org.apache.spark.SparkConf import org.apache.spark.sql._ @@ -31,7 +31,6 @@ class GlutenSessionExtensionSuite extends GlutenSQLTestsTrait { testGluten("test gluten extensions") { assert(spark.sessionState.columnarRules.contains(ColumnarOverrideRules(spark))) - assert(spark.sessionState.planner.strategies.contains(JoinSelectionOverrides(spark))) assert(spark.sessionState.planner.strategies.contains(MySparkStrategy(spark))) assert(spark.sessionState.analyzer.extendedResolutionRules.contains(MyRule(spark))) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 6162b5651980..e54aca34ec75 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -866,7 +866,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("Shouldn't change broadcast join buildSide if user clearly specified") .exclude("Shouldn't bias towards build right if user didn't specify") .exclude("SPARK-23192: broadcast hint should be retained after using the cached data") - .exclude("broadcast hint isn't propagated after a join") .exclude("broadcast join where streamed side's output partitioning is HashPartitioning") enableSuite[GlutenExistenceJoinSuite] @@ -1136,9 +1135,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenJoinSuite] // exclude as it check spark plan .exclude("SPARK-36794: Ignore duplicated key when building relation for semi/anti hash join") - // exclude as it check for SMJ node - .exclude( - "SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)") enableSuite[GlutenMathFunctionsSuite] enableSuite[GlutenMetadataCacheSuite] .exclude("SPARK-16336,SPARK-27961 Suggest fixing FileNotFoundException") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala index 09718fb1a439..5ef4056201ed 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec - class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait { override def testNameBlackList: Seq[String] = Seq( @@ -57,14 +54,4 @@ class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait { |""".stripMargin checkAnswer(spark.sql(sql), Seq(Row(0, 1), Row(1, 2), Row(2, 3))) } - - testGluten( - "SPARK-43113: Full outer join with duplicate stream-side" + - " references in condition (SHJ)") { - def check(plan: SparkPlan): Unit = { - assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1) - } - - dupStreamSideColTest("MERGE", check) - } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala index 8d795bbffea0..332c21418a9b 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala @@ -84,11 +84,11 @@ class GlutenReplaceHashWithSortAggSuite withTempView("t1", "t2") { spark.range(100).selectExpr("id as key").createOrReplaceTempView("t1") spark.range(50).selectExpr("id as key").createOrReplaceTempView("t2") - Seq("COUNT", "COLLECT_LIST").foreach { - aggExpr => + Seq(("COUNT", 0, 1, 2, 0), ("COLLECT_LIST", 2, 0, 2, 0)).foreach { + aggExprInfo => val query = s""" - |SELECT key, $aggExpr(key) + |SELECT key, ${aggExprInfo._1}(key) |FROM |( | SELECT /*+ SHUFFLE_MERGE(t1) */ t1.key AS key @@ -98,7 +98,7 @@ class GlutenReplaceHashWithSortAggSuite |) |GROUP BY key """.stripMargin - checkAggs(query, 2, 0, 2, 0) + checkAggs(query, aggExprInfo._2, aggExprInfo._3, aggExprInfo._4, aggExprInfo._5) } } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala index 00b4bf5821f1..4df9c63b3ef6 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.extension.{ColumnarOverrideRules, JoinSelectionOverrides} +import org.apache.gluten.extension.ColumnarOverrideRules import org.apache.spark.SparkConf import org.apache.spark.sql._ @@ -31,7 +31,6 @@ class GlutenSessionExtensionSuite extends GlutenSQLTestsTrait { testGluten("test gluten extensions") { assert(spark.sessionState.columnarRules.contains(ColumnarOverrideRules(spark))) - assert(spark.sessionState.planner.strategies.contains(JoinSelectionOverrides(spark))) assert(spark.sessionState.planner.strategies.contains(MySparkStrategy(spark))) assert(spark.sessionState.analyzer.extendedResolutionRules.contains(MyRule(spark))) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index ec80ba86a7b9..4ef96bec27eb 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -153,9 +153,6 @@ class GlutenConfig(conf: SQLConf) extends Logging { def logicalJoinOptimizationThrottle: Integer = conf.getConf(COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_THROTTLE) - def enableLogicalJoinOptimize: Boolean = - conf.getConf(COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_ENABLED) - def enableScanOnly: Boolean = conf.getConf(COLUMNAR_SCAN_ONLY_ENABLED) def tmpFile: Option[String] = conf.getConf(COLUMNAR_TEMP_DIR) @@ -1007,13 +1004,6 @@ object GlutenConfig { .intConf .createWithDefault(12) - val COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_ENABLED = - buildConf("spark.gluten.sql.columnar.logicalJoinOptimizeEnable") - .internal() - .doc("Enable or disable columnar logicalJoinOptimize.") - .booleanConf - .createWithDefault(false) - val COLUMNAR_SCAN_ONLY_ENABLED = buildConf("spark.gluten.sql.columnar.scanOnly") .internal() diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala deleted file mode 100644 index 20b9dea333a5..000000000000 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys -import org.apache.spark.sql.catalyst.plans.JoinType -import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint, LogicalPlan} - -// https://issues.apache.org/jira/browse/SPARK-36745 -object JoinSelectionShim { - object ExtractEquiJoinKeysShim { - type ReturnType = - ( - JoinType, - Seq[Expression], - Seq[Expression], - Option[Expression], - LogicalPlan, - LogicalPlan, - JoinHint) - def unapply(join: Join): Option[ReturnType] = { - ExtractEquiJoinKeys.unapply(join).map { - case ( - joinType, - leftKeys, - rightKeys, - otherPredicates, - predicatesOfJoinKeys, - left, - right, - hint) => - (joinType, leftKeys, rightKeys, otherPredicates, left, right, hint) - } - } - } -} diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala deleted file mode 100644 index 20b9dea333a5..000000000000 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys -import org.apache.spark.sql.catalyst.plans.JoinType -import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint, LogicalPlan} - -// https://issues.apache.org/jira/browse/SPARK-36745 -object JoinSelectionShim { - object ExtractEquiJoinKeysShim { - type ReturnType = - ( - JoinType, - Seq[Expression], - Seq[Expression], - Option[Expression], - LogicalPlan, - LogicalPlan, - JoinHint) - def unapply(join: Join): Option[ReturnType] = { - ExtractEquiJoinKeys.unapply(join).map { - case ( - joinType, - leftKeys, - rightKeys, - otherPredicates, - predicatesOfJoinKeys, - left, - right, - hint) => - (joinType, leftKeys, rightKeys, otherPredicates, left, right, hint) - } - } - } -} diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala deleted file mode 100644 index 20b9dea333a5..000000000000 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/JoinSelectionShim.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys -import org.apache.spark.sql.catalyst.plans.JoinType -import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint, LogicalPlan} - -// https://issues.apache.org/jira/browse/SPARK-36745 -object JoinSelectionShim { - object ExtractEquiJoinKeysShim { - type ReturnType = - ( - JoinType, - Seq[Expression], - Seq[Expression], - Option[Expression], - LogicalPlan, - LogicalPlan, - JoinHint) - def unapply(join: Join): Option[ReturnType] = { - ExtractEquiJoinKeys.unapply(join).map { - case ( - joinType, - leftKeys, - rightKeys, - otherPredicates, - predicatesOfJoinKeys, - left, - right, - hint) => - (joinType, leftKeys, rightKeys, otherPredicates, left, right, hint) - } - } - } -} From f80c06892c4604b69f822b435bb7c6458f46ac76 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Mon, 1 Jul 2024 08:17:52 +0700 Subject: [PATCH 365/402] [CELEBORN] Avoid CelebornShuffleManager#getWriter adding shuffle id repeatedly to columnarShuffleIds (#6281) --- .../shuffle/gluten/celeborn/CelebornShuffleManager.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index 63fb0cc1b9bd..e61aeb3d7088 100644 --- a/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -215,12 +215,8 @@ public ShuffleHandle registerShuffle( @Override public boolean unregisterShuffle(int shuffleId) { - if (columnarShuffleIds.contains(shuffleId)) { - if (columnarShuffleManager().unregisterShuffle(shuffleId)) { - return columnarShuffleIds.remove(shuffleId); - } else { - return false; - } + if (columnarShuffleIds.remove(shuffleId)) { + return columnarShuffleManager().unregisterShuffle(shuffleId); } return CelebornUtils.unregisterShuffle( lifecycleManager, @@ -311,7 +307,6 @@ public ShuffleWriter getWriter( return vanillaCelebornShuffleManager().getWriter(handle, mapId, context, metrics); } } else { - columnarShuffleIds.add(handle.shuffleId()); return columnarShuffleManager().getWriter(handle, mapId, context, metrics); } } catch (Exception e) { From b40989a4ea24ae9a06e9209f9e6fd772d70c65b2 Mon Sep 17 00:00:00 2001 From: exmy Date: Mon, 1 Jul 2024 10:30:05 +0800 Subject: [PATCH 366/402] [CH] Support bit_get/bit_count function (#5636) What changes were proposed in this pull request? How was this patch tested? Pass CI --- .../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 7 +++++++ cpp-ch/local-engine/Parser/SerializedPlanParser.h | 2 ++ .../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 +- .../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 +- .../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 +- .../gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 +- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 188995f11058..c0f37b08616e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -1326,6 +1326,13 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr } } + test("bit_get/bit_count") { + runQueryAndCompare( + "select bit_count(id), bit_get(id, 0), bit_get(id, 1), bit_get(id, 2), bit_get(id, 3) from range(100)") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("test 'EqualNullSafe'") { runQueryAndCompare("select l_linenumber <=> l_orderkey, l_linenumber <=> null from lineitem") { checkGlutenOperatorMatch[ProjectExecTransformer] diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 1785f64ee17c..4a4a19af30b0 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -94,6 +94,8 @@ static const std::map SCALAR_FUNCTIONS {"bitwise_and", "bitAnd"}, {"bitwise_or", "bitOr"}, {"bitwise_xor", "bitXor"}, + {"bit_get", "bitTest"}, + {"bit_count", "bitCount"}, {"sqrt", "sqrt"}, {"cbrt", "cbrt"}, {"degrees", "degrees"}, diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 3048c3f9cab5..72185e8cef10 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -561,7 +561,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-28322: IntegralDivide supports decimal type") .exclude("SPARK-33008: division by zero on divide-like operations returns incorrect result") .exclude("SPARK-34920: error class") - enableSuite[GlutenBitwiseExpressionsSuite].exclude("BitGet") + enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenCastSuite] .exclude("null cast") .exclude("cast string to date") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 769707d4eb5f..2df825fc74bf 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -546,7 +546,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-36920: Support day-time intervals by ABS") .exclude("SPARK-36921: Support YearMonthIntervalType by div") .exclude("SPARK-36921: Support DayTimeIntervalType by div") - enableSuite[GlutenBitwiseExpressionsSuite].exclude("BitGet") + enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenCastSuite] .exclude("null cast") .exclude("cast string to date") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 268f22fe6981..edca8f5284f6 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -488,7 +488,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-36920: Support day-time intervals by ABS") .exclude("SPARK-36921: Support YearMonthIntervalType by div") .exclude("SPARK-36921: Support DayTimeIntervalType by div") - enableSuite[GlutenBitwiseExpressionsSuite].exclude("BitGet") + enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenCastSuite] .exclude("null cast") .exclude("cast string to date") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 268f22fe6981..edca8f5284f6 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -488,7 +488,7 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("SPARK-36920: Support day-time intervals by ABS") .exclude("SPARK-36921: Support YearMonthIntervalType by div") .exclude("SPARK-36921: Support DayTimeIntervalType by div") - enableSuite[GlutenBitwiseExpressionsSuite].exclude("BitGet") + enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenCastSuite] .exclude("null cast") .exclude("cast string to date") From 231f00c71454837717b27374267149d52b5b9833 Mon Sep 17 00:00:00 2001 From: exmy Date: Mon, 1 Jul 2024 10:35:31 +0800 Subject: [PATCH 367/402] [CH] Support bit_length/octet_length function (#6259) What changes were proposed in this pull request? support bit_length/octet_length function move length impl to funciton parser These three functions may receive int type argument in spark ut, add a cast to string. How was this patch tested? PASS CI --- .../gluten/utils/CHExpressionUtil.scala | 1 - .../Parser/SerializedPlanParser.cpp | 9 --- .../Parser/SerializedPlanParser.h | 3 +- .../scalar_function_parser/bitLength.cpp | 65 ++++++++++++++++ .../Parser/scalar_function_parser/length.cpp | 74 +++++++++++++++++++ .../scalar_function_parser/octetLength.cpp | 60 +++++++++++++++ .../expression/ExpressionMappings.scala | 1 + .../clickhouse/ClickHouseTestSettings.scala | 4 +- .../sql/GlutenStringFunctionsSuite.scala | 7 +- .../clickhouse/ClickHouseTestSettings.scala | 4 +- .../sql/GlutenStringFunctionsSuite.scala | 4 - .../clickhouse/ClickHouseTestSettings.scala | 4 +- .../sql/GlutenStringFunctionsSuite.scala | 4 - .../clickhouse/ClickHouseTestSettings.scala | 4 +- .../sql/GlutenStringFunctionsSuite.scala | 4 - .../gluten/expression/ExpressionNames.scala | 1 + 16 files changed, 215 insertions(+), 34 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/bitLength.cpp create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/length.cpp create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/octetLength.cpp diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index e9bee84396f8..14f0ff489188 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -194,7 +194,6 @@ object CHExpressionUtil { URL_ENCODE -> DefaultValidator(), SKEWNESS -> DefaultValidator(), SOUNDEX -> DefaultValidator(), - BIT_LENGTH -> DefaultValidator(), MAKE_YM_INTERVAL -> DefaultValidator(), MAP_ZIP_WITH -> DefaultValidator(), ZIP_WITH -> DefaultValidator(), diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 325ec32dc65f..77819fd73e75 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -648,15 +648,6 @@ SerializedPlanParser::getFunctionName(const std::string & function_signature, co if (null_on_overflow) ch_function_name = ch_function_name + "OrNull"; } - else if (function_name == "char_length") - { - /// In Spark - /// char_length returns the number of bytes when input is binary type, corresponding to CH length function - /// char_length returns the number of characters when input is string type, corresponding to CH char_length function - ch_function_name = SCALAR_FUNCTIONS.at(function_name); - if (function_signature.find("vbin") != std::string::npos) - ch_function_name = "length"; - } else if (function_name == "reverse") { if (function.output_type().has_list()) diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 4a4a19af30b0..ad2b0d50ec6a 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -130,8 +130,6 @@ static const std::map SCALAR_FUNCTIONS {"ltrim", ""}, // trimRight or trimRightSpark, depends on argument size {"rtrim", ""}, // trimBoth or trimBothSpark, depends on argument size {"strpos", "positionUTF8"}, - {"char_length", - "char_length"}, /// Notice: when input argument is binary type, corresponding ch function is length instead of char_length {"replace", "replaceAll"}, {"regexp_replace", "replaceRegexpAll"}, {"regexp_extract_all", "regexpExtractAllSpark"}, @@ -306,6 +304,7 @@ class SerializedPlanParser std::shared_ptr expressionsToActionsDAG( const std::vector & expressions, const DB::Block & header, const DB::Block & read_schema); RelMetricPtr getMetric() { return metrics.empty() ? nullptr : metrics.at(0); } + const std::unordered_map & getFunctionMapping() { return function_mapping; } static std::string getFunctionName(const std::string & function_sig, const substrait::Expression_ScalarFunction & function); diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/bitLength.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/bitLength.cpp new file mode 100644 index 000000000000..9358c45788cf --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/bitLength.cpp @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class FunctionParserBitLength : public FunctionParser +{ +public: + explicit FunctionParserBitLength(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } + ~FunctionParserBitLength() override = default; + + static constexpr auto name = "bit_length"; + + String getName() const override { return name; } + + const ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const override + { + // parse bit_length(a) as octet_length(a) * 8 + auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); + if (parsed_args.size() != 1) + throw Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly one arguments", getName()); + + const auto * arg = parsed_args[0]; + const auto * new_arg = arg; + if (isInt(DB::removeNullable(arg->result_type))) + { + const auto * string_type_node = addColumnToActionsDAG(actions_dag, std::make_shared(), "Nullable(String)"); + new_arg = toFunctionNode(actions_dag, "CAST", {arg, string_type_node}); + } + + const auto * octet_length_node = toFunctionNode(actions_dag, "octet_length", {new_arg}); + const auto * const_eight_node = addColumnToActionsDAG(actions_dag, std::make_shared(), 8); + const auto * result_node = toFunctionNode(actions_dag, "multiply", {octet_length_node, const_eight_node}); + + return convertNodeTypeIfNeeded(substrait_func, result_node, actions_dag);; + } +}; + +static FunctionParserRegister register_bit_length; +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/length.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/length.cpp new file mode 100644 index 000000000000..85fe1f29aa25 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/length.cpp @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class FunctionParserLength : public FunctionParser +{ +public: + explicit FunctionParserLength(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } + ~FunctionParserLength() override = default; + + static constexpr auto name = "char_length"; + + String getName() const override { return name; } + + const ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const override + { + /** + parse length(a) as + if input is binary type + length(a) as length(a) + else + length(a) as char_length(a) + */ + auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); + if (parsed_args.size() != 1) + throw Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly one arguments", getName()); + + const auto * arg = parsed_args[0]; + const auto * new_arg = arg; + if (isInt(removeNullable(arg->result_type))) + { + const auto * string_type_node = addColumnToActionsDAG(actions_dag, std::make_shared(), "Nullable(String)"); + new_arg = toFunctionNode(actions_dag, "CAST", {arg, string_type_node}); + } + + auto function_signature = plan_parser->getFunctionMapping().at(std::to_string(substrait_func.function_reference())); + const ActionsDAG::Node * result_node; + if (function_signature.find("vbin") != std::string::npos) + result_node = toFunctionNode(actions_dag, "length", {new_arg}); + else + result_node = toFunctionNode(actions_dag, "char_length", {new_arg}); + + return convertNodeTypeIfNeeded(substrait_func, result_node, actions_dag);; + } +}; + +static FunctionParserRegister register_length; +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/octetLength.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/octetLength.cpp new file mode 100644 index 000000000000..52cbd0317290 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/octetLength.cpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} +} + +namespace local_engine +{ +class FunctionParserOctetLength : public FunctionParser +{ +public: + explicit FunctionParserOctetLength(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) { } + ~FunctionParserOctetLength() override = default; + + static constexpr auto name = "octet_length"; + + String getName() const override { return name; } + + const ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, ActionsDAGPtr & actions_dag) const override + { + auto parsed_args = parseFunctionArguments(substrait_func, "", actions_dag); + if (parsed_args.size() != 1) + throw Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires exactly one arguments", getName()); + + const auto * arg = parsed_args[0]; + const auto * new_arg = arg; + if (isInt(DB::removeNullable(arg->result_type))) + { + const auto * string_type_node = addColumnToActionsDAG(actions_dag, std::make_shared(), "Nullable(String)"); + new_arg = toFunctionNode(actions_dag, "CAST", {arg, string_type_node}); + } + const auto * octet_length_node = toFunctionNode(actions_dag, "octet_length", {new_arg}); + return convertNodeTypeIfNeeded(substrait_func, octet_length_node, actions_dag);; + } +}; + +static FunctionParserRegister register_octet_length; +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 678ba38172eb..806ec844de60 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -101,6 +101,7 @@ object ExpressionMappings { Sig[Encode](ENCODE), Sig[Uuid](UUID), Sig[BitLength](BIT_LENGTH), + Sig[OctetLength](OCTET_LENGTH), Sig[Levenshtein](LEVENSHTEIN), Sig[UnBase64](UNBASE64), Sig[Base64](BASE64), diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 72185e8cef10..60df3ee37f66 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -438,6 +438,9 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string overlay function") .exclude("binary overlay function") .exclude("string parse_url function") + .exclude("string / binary length function") + .exclude("SPARK-36751: add octet length api for scala") + .exclude("SPARK-36751: add bit length api for scala") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") .exclude("SPARK-26893: Allow pushdown of partition pruning subquery filters to file source") @@ -905,7 +908,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("LOCATE") .exclude("LPAD/RPAD") .exclude("REPEAT") - .exclude("length for string / binary") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") .excludeGlutenTest("SPARK-40213: ascii for Latin-1 Supplement characters") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index a686b6456e9f..b88fdc59db6a 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -21,9 +21,4 @@ import org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper class GlutenStringFunctionsSuite extends StringFunctionsSuite with GlutenSQLTestsTrait - with ExpressionEvalHelper { - - override def testNameBlackList: Seq[String] = super.testNameBlackList ++ Seq( - "string / binary length function" - ) -} + with ExpressionEvalHelper {} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 2df825fc74bf..df9f49bfc72e 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -459,6 +459,9 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string overlay function") .exclude("binary overlay function") .exclude("string parse_url function") + .exclude("string / binary length function") + .exclude("SPARK-36751: add octet length api for scala") + .exclude("SPARK-36751: add bit length api for scala") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") .exclude("SPARK-26893: Allow pushdown of partition pruning subquery filters to file source") @@ -864,7 +867,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("translate") .exclude("LOCATE") .exclude("REPEAT") - .exclude("length for string / binary") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") enableSuite[GlutenTryCastSuite] diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index c58284e4403b..3d82e214f031 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -30,10 +30,6 @@ class GlutenStringFunctionsSuite import testImplicits._ - override def testNameBlackList: Seq[String] = super.testNameBlackList ++ Seq( - "string / binary length function" - ) - testGluten("string split function with no limit and regex pattern") { val df1 = Seq(("aaAbbAcc4")).toDF("a").select(split($"a", "A")) checkAnswer(df1, Row(Seq("aa", "bb", "cc4"))) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index edca8f5284f6..0dc2cdd89f93 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -458,6 +458,9 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string overlay function") .exclude("binary overlay function") .exclude("string parse_url function") + .exclude("string / binary length function") + .exclude("SPARK-36751: add octet length api for scala") + .exclude("SPARK-36751: add bit length api for scala") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") .exclude("SPARK-26893: Allow pushdown of partition pruning subquery filters to file source") @@ -768,7 +771,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("translate") .exclude("LOCATE") .exclude("REPEAT") - .exclude("length for string / binary") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index c58284e4403b..3d82e214f031 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -30,10 +30,6 @@ class GlutenStringFunctionsSuite import testImplicits._ - override def testNameBlackList: Seq[String] = super.testNameBlackList ++ Seq( - "string / binary length function" - ) - testGluten("string split function with no limit and regex pattern") { val df1 = Seq(("aaAbbAcc4")).toDF("a").select(split($"a", "A")) checkAnswer(df1, Row(Seq("aa", "bb", "cc4"))) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index edca8f5284f6..0dc2cdd89f93 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -458,6 +458,9 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("string overlay function") .exclude("binary overlay function") .exclude("string parse_url function") + .exclude("string / binary length function") + .exclude("SPARK-36751: add octet length api for scala") + .exclude("SPARK-36751: add bit length api for scala") enableSuite[GlutenSubquerySuite] .exclude("SPARK-15370: COUNT bug in subquery in subquery in subquery") .exclude("SPARK-26893: Allow pushdown of partition pruning subquery filters to file source") @@ -768,7 +771,6 @@ class ClickHouseTestSettings extends BackendTestSettings { .exclude("translate") .exclude("LOCATE") .exclude("REPEAT") - .exclude("length for string / binary") .exclude("ParseUrl") .exclude("SPARK-33468: ParseUrl in ANSI mode should fail if input string is not a valid url") enableSuite[GlutenDataSourceV2DataFrameSessionCatalogSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala index c58284e4403b..3d82e214f031 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenStringFunctionsSuite.scala @@ -30,10 +30,6 @@ class GlutenStringFunctionsSuite import testImplicits._ - override def testNameBlackList: Seq[String] = super.testNameBlackList ++ Seq( - "string / binary length function" - ) - testGluten("string split function with no limit and regex pattern") { val df1 = Seq(("aaAbbAcc4")).toDF("a").select(split($"a", "A")) checkAnswer(df1, Row(Seq("aa", "bb", "cc4"))) diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 87b1b4e7539b..7060e297ea10 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -127,6 +127,7 @@ object ExpressionNames { final val ENCODE = "encode" final val UUID = "uuid" final val BIT_LENGTH = "bit_length" + final val OCTET_LENGTH = "octet_length" final val LEVENSHTEIN = "levenshteinDistance" final val UNBASE64 = "unbase64" final val BASE64 = "base64" From ae2d1255739a0a4c37f35d3a938c2af30849aeae Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 1 Jul 2024 11:36:18 +0800 Subject: [PATCH 368/402] [CORE] Execution runtime / native memory manager refactor (#6243) --- .github/workflows/velox_docker.yml | 27 +- .../alloc/CHNativeMemoryAllocators.java | 13 +- .../shuffle/CHColumnarShuffleWriter.scala | 8 +- .../memory/TestTaskMemoryManagerSuite.java | 3 +- .../gluten/utils/VeloxBatchAppender.java | 12 +- .../utils/VeloxBatchAppenderJniWrapper.java | 5 +- .../apache/gluten/utils/VeloxBloomFilter.java | 8 +- .../utils/VeloxBloomFilterJniWrapper.java | 5 +- .../velox/VeloxSparkPlanExecApi.scala | 26 +- .../velox/VeloxTransformerApi.scala | 4 +- .../execution/RowToVeloxColumnarExec.scala | 16 +- .../execution/VeloxColumnarToRowExec.scala | 27 +- .../apache/gluten/utils/DatasourceUtil.scala | 9 +- .../spark/sql/execution/BroadcastUtils.scala | 12 +- .../ColumnarCachedBatchSerializer.scala | 32 +- .../datasources/velox/VeloxBlockStripes.java | 3 +- .../velox/VeloxFormatWriterInjects.scala | 22 +- cpp/core/compute/Runtime.cc | 7 +- cpp/core/compute/Runtime.h | 52 ++- cpp/core/jni/JniCommon.cc | 2 +- cpp/core/jni/JniCommon.h | 3 +- cpp/core/jni/JniError.cc | 7 + cpp/core/jni/JniError.h | 13 +- cpp/core/jni/JniWrapper.cc | 390 +++++++----------- cpp/core/tests/CMakeLists.txt | 1 + cpp/core/tests/ObjectStoreTest.cc | 116 ++++++ cpp/core/utils/ObjectStore.cc | 16 +- cpp/core/utils/ObjectStore.h | 64 ++- cpp/core/utils/ResourceMap.h | 40 +- cpp/velox/benchmarks/GenericBenchmark.cc | 104 ++--- cpp/velox/benchmarks/ParquetWriteBenchmark.cc | 2 +- cpp/velox/compute/VeloxBackend.cc | 6 +- cpp/velox/compute/VeloxRuntime.cc | 73 ++-- cpp/velox/compute/VeloxRuntime.h | 59 +-- cpp/velox/compute/WholeStageResultIterator.cc | 7 +- cpp/velox/jni/VeloxJniWrapper.cc | 27 +- cpp/velox/memory/VeloxMemoryManager.cc | 21 +- cpp/velox/memory/VeloxMemoryManager.h | 10 +- cpp/velox/tests/MemoryManagerTest.cc | 2 +- cpp/velox/tests/RuntimeTest.cc | 74 ++-- ep/build-velox/src/build_velox.sh | 2 +- ...lebornHashBasedColumnarShuffleWriter.scala | 12 +- ...VeloxCelebornColumnarBatchSerializer.scala | 13 +- ...lebornHashBasedColumnarShuffleWriter.scala | 56 ++- .../memory/memtarget/MemoryTargets.java | 5 +- .../gluten/memory/memtarget/Spiller.java | 9 +- .../gluten/memory/memtarget/Spillers.java | 39 +- .../memory/memtarget/TreeMemoryTarget.java | 5 +- .../memory/memtarget/TreeMemoryTargets.java | 41 +- .../spark/RegularMemoryConsumer.java | 12 +- .../memtarget/spark/TreeMemoryConsumer.java | 15 +- .../memtarget/spark/TreeMemoryConsumers.java | 8 +- .../apache/spark/memory/SparkMemoryUtil.scala | 4 + .../spark/TreeMemoryConsumerTest.java | 9 +- .../ColumnarBatchJniWrapper.java | 22 +- .../gluten/columnarbatch/ColumnarBatches.java | 50 +-- .../gluten/columnarbatch/IndicatorVector.java | 129 +----- .../columnarbatch/IndicatorVectorBase.java | 166 ++++++++ .../datasource/DatasourceJniWrapper.java | 16 +- .../apache/gluten/exec/RuntimeJniWrapper.java | 12 +- .../memory/alloc/NativeMemoryAllocator.java | 49 --- .../memory/alloc/NativeMemoryAllocators.java | 54 --- .../arrow/alloc/ArrowBufferAllocators.java | 6 +- .../ManagedReservationListener.java | 2 +- .../ReservationListener.java | 7 +- .../memory/listener/ReservationListeners.java | 77 ++++ .../memory/nmm/NativeMemoryManager.java | 124 ------ .../memory/nmm/NativeMemoryManagers.java | 167 -------- .../vectorized/ColumnarBatchInIterator.java | 5 +- .../vectorized/ColumnarBatchOutIterator.java | 10 +- .../ColumnarBatchSerializerJniWrapper.java | 11 +- .../NativeColumnarToRowJniWrapper.java | 9 +- .../vectorized/NativePlanEvaluator.java | 55 +-- .../NativeRowToColumnarJniWrapper.java | 7 +- .../vectorized/PlanEvaluatorJniWrapper.java | 7 +- .../vectorized/ShuffleReaderJniWrapper.java | 6 +- .../vectorized/ShuffleWriterJniWrapper.java | 10 +- .../org/apache/gluten/exec/Runtime.scala | 116 +++++- .../org/apache/gluten/exec/Runtimes.scala | 11 +- .../vectorized/ColumnarBatchSerializer.scala | 17 +- .../spark/shuffle/ColumnarShuffleWriter.scala | 47 +-- .../execution/ColumnarBuildSideRelation.scala | 26 +- .../spark/sql/execution/utils/ExecUtil.scala | 17 +- .../VeloxUniffleColumnarShuffleWriter.java | 58 ++- 84 files changed, 1304 insertions(+), 1547 deletions(-) create mode 100644 cpp/core/tests/ObjectStoreTest.cc create mode 100644 gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java delete mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java delete mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java rename gluten-data/src/main/java/org/apache/gluten/memory/{nmm => listener}/ManagedReservationListener.java (98%) rename gluten-data/src/main/java/org/apache/gluten/memory/{nmm => listener}/ReservationListener.java (76%) create mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java delete mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java delete mode 100644 gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index fd937f6c1d49..098b2a2d57eb 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -292,7 +292,7 @@ jobs: - name: TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory, memory isolation off run: | cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \ + && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ --skip-data-gen -m=OffHeapExecutionMemory \ -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ @@ -304,7 +304,7 @@ jobs: - name: TPC-DS SF30.0 Parquet local spark3.2 Q67 low memory, memory isolation on run: | cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \ + && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ --skip-data-gen -m=OffHeapExecutionMemory \ -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ @@ -315,7 +315,7 @@ jobs: - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q95 low memory, memory isolation on run: | cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \ + && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ --skip-data-gen -m=OffHeapExecutionMemory \ -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ @@ -326,10 +326,21 @@ jobs: - name: TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory run: | cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \ + && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ --skip-data-gen -m=OffHeapExecutionMemory \ -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ + -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ + -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ + -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ + -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 + - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory, memory isolation on + if: false # Disabled as error https://gist.github.com/zhztheplayer/abd5e83ccdc48730678ae7ebae479fcc + run: | + cd tools/gluten-it \ + && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ + --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ + --skip-data-gen -m=OffHeapExecutionMemory \ -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ @@ -338,7 +349,7 @@ jobs: - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q97 low memory # The case currently causes crash with "free: invalid size". run: | cd tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \ + && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q97 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ --skip-data-gen -m=OffHeapExecutionMemory \ -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ @@ -455,7 +466,7 @@ jobs: strategy: fail-fast: false matrix: - spark: ["spark-3.2"] + spark: [ "spark-3.2" ] runs-on: ubuntu-20.04 container: centos:8 steps: @@ -520,8 +531,8 @@ jobs: strategy: fail-fast: false matrix: - spark: ["spark-3.2"] - celeborn: ["celeborn-0.4.1", "celeborn-0.3.2-incubating"] + spark: [ "spark-3.2" ] + celeborn: [ "celeborn-0.4.1", "celeborn-0.3.2-incubating" ] runs-on: ubuntu-20.04 container: ubuntu:22.04 steps: diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java index 01fb7e3e2f7c..0f30972fcd44 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java @@ -19,13 +19,12 @@ import org.apache.gluten.memory.SimpleMemoryUsageRecorder; import org.apache.gluten.memory.memtarget.MemoryTargets; import org.apache.gluten.memory.memtarget.Spiller; +import org.apache.gluten.memory.memtarget.Spillers; import org.apache.spark.memory.TaskMemoryManager; import org.apache.spark.util.TaskResources; -import java.util.Arrays; import java.util.Collections; -import java.util.List; /** * Built-in toolkit for managing native memory allocations. To use the facility, one should import @@ -46,12 +45,12 @@ private CHNativeMemoryAllocators() {} private static CHNativeMemoryAllocatorManager createNativeMemoryAllocatorManager( String name, TaskMemoryManager taskMemoryManager, - List spillers, + Spiller spiller, SimpleMemoryUsageRecorder usage) { CHManagedCHReservationListener rl = new CHManagedCHReservationListener( - MemoryTargets.newConsumer(taskMemoryManager, name, spillers, Collections.emptyMap()), + MemoryTargets.newConsumer(taskMemoryManager, name, spiller, Collections.emptyMap()), usage); return new CHNativeMemoryAllocatorManagerImpl(CHNativeMemoryAllocator.createListenable(rl)); } @@ -67,7 +66,7 @@ public static CHNativeMemoryAllocator contextInstance() { createNativeMemoryAllocatorManager( "ContextInstance", TaskResources.getLocalTaskContext().taskMemoryManager(), - Collections.emptyList(), + Spillers.NOOP, TaskResources.getSharedUsage()); TaskResources.addResource(id, manager); } @@ -78,7 +77,7 @@ public static CHNativeMemoryAllocator contextInstanceForUT() { return CHNativeMemoryAllocator.getDefaultForUT(); } - public static CHNativeMemoryAllocator createSpillable(String name, Spiller... spillers) { + public static CHNativeMemoryAllocator createSpillable(String name, Spiller spiller) { if (!TaskResources.inSparkTask()) { throw new IllegalStateException("spiller must be used in a Spark task"); } @@ -87,7 +86,7 @@ public static CHNativeMemoryAllocator createSpillable(String name, Spiller... sp createNativeMemoryAllocatorManager( name, TaskResources.getLocalTaskContext().taskMemoryManager(), - Arrays.asList(spillers), + spiller, TaskResources.getSharedUsage()); TaskResources.addAnonymousResource(manager); // force add memory consumer to task memory manager, will release by inactivate diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala index 4a1adbec7418..c113f8d4dd31 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala @@ -29,7 +29,6 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.{SparkDirectoryUtil, Utils} import java.io.IOException -import java.util import java.util.{Locale, UUID} class CHColumnarShuffleWriter[K, V]( @@ -122,7 +121,10 @@ class CHColumnarShuffleWriter[K, V]( CHNativeMemoryAllocators.createSpillable( "ShuffleWriter", new Spiller() { - override def spill(self: MemoryTarget, size: Long): Long = { + override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = { + if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { + return 0L; + } if (nativeSplitter == 0) { throw new IllegalStateException( "Fatal: spill() called before a shuffle writer " + @@ -134,8 +136,6 @@ class CHColumnarShuffleWriter[K, V]( logError(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data") spilled } - - override def applicablePhases(): util.Set[Spiller.Phase] = Spillers.PHASE_SET_SPILL_ONLY } ) } diff --git a/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java b/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java index b575de403bfd..905ffacde023 100644 --- a/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java +++ b/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java @@ -21,6 +21,7 @@ import org.apache.gluten.memory.alloc.CHNativeMemoryAllocator; import org.apache.gluten.memory.alloc.CHNativeMemoryAllocatorManagerImpl; import org.apache.gluten.memory.memtarget.MemoryTargets; +import org.apache.gluten.memory.memtarget.Spillers; import org.apache.spark.SparkConf; import org.apache.spark.internal.config.package$; @@ -52,7 +53,7 @@ public void initMemoryManager() { listener = new CHManagedCHReservationListener( MemoryTargets.newConsumer( - taskMemoryManager, "test", Collections.emptyList(), Collections.emptyMap()), + taskMemoryManager, "test", Spillers.NOOP, Collections.emptyMap()), new SimpleMemoryUsageRecorder()); manager = new CHNativeMemoryAllocatorManagerImpl(new CHNativeMemoryAllocator(-1L, listener)); diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java index 1bf34b5ce3b6..32b2289471f9 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java @@ -18,8 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.Runtimes; -import org.apache.gluten.memory.nmm.NativeMemoryManager; -import org.apache.gluten.memory.nmm.NativeMemoryManagers; import org.apache.gluten.vectorized.ColumnarBatchInIterator; import org.apache.gluten.vectorized.ColumnarBatchOutIterator; @@ -30,12 +28,10 @@ public final class VeloxBatchAppender { public static ColumnarBatchOutIterator create( int minOutputBatchSize, Iterator in) { - final Runtime runtime = Runtimes.contextInstance(); - final NativeMemoryManager nmm = NativeMemoryManagers.contextInstance("VeloxBatchAppender"); + final Runtime runtime = Runtimes.contextInstance("VeloxBatchAppender"); long outHandle = - VeloxBatchAppenderJniWrapper.forRuntime(runtime) - .create( - nmm.getNativeInstanceHandle(), minOutputBatchSize, new ColumnarBatchInIterator(in)); - return new ColumnarBatchOutIterator(runtime, outHandle, nmm); + VeloxBatchAppenderJniWrapper.create(runtime) + .create(minOutputBatchSize, new ColumnarBatchInIterator(in)); + return new ColumnarBatchOutIterator(runtime, outHandle); } } diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java index 9e2531951ccc..231e65553981 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java @@ -27,7 +27,7 @@ private VeloxBatchAppenderJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static VeloxBatchAppenderJniWrapper forRuntime(Runtime runtime) { + public static VeloxBatchAppenderJniWrapper create(Runtime runtime) { return new VeloxBatchAppenderJniWrapper(runtime); } @@ -36,6 +36,5 @@ public long handle() { return runtime.getHandle(); } - public native long create( - long memoryManagerHandle, int minOutputBatchSize, ColumnarBatchInIterator itr); + public native long create(int minOutputBatchSize, ColumnarBatchInIterator itr); } diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java index 13ba8e0113b1..f23426d7da9d 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java @@ -16,6 +16,8 @@ */ package org.apache.gluten.utils; +import org.apache.gluten.exec.Runtimes; + import org.apache.commons.io.IOUtils; import org.apache.spark.util.sketch.BloomFilter; import org.apache.spark.util.sketch.IncompatibleMergeException; @@ -27,17 +29,15 @@ import java.io.OutputStream; public class VeloxBloomFilter extends BloomFilter { - - private final VeloxBloomFilterJniWrapper jni; + private final VeloxBloomFilterJniWrapper jni = + VeloxBloomFilterJniWrapper.create(Runtimes.contextInstance("VeloxBloomFilter")); private final long handle; private VeloxBloomFilter(byte[] data) { - jni = VeloxBloomFilterJniWrapper.create(); handle = jni.init(data); } private VeloxBloomFilter(int capacity) { - jni = VeloxBloomFilterJniWrapper.create(); handle = jni.empty(capacity); } diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java index 94f8e17bc6e5..3ddfd2c02ed8 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; public class VeloxBloomFilterJniWrapper implements RuntimeAware { private final Runtime runtime; @@ -27,8 +26,8 @@ private VeloxBloomFilterJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static VeloxBloomFilterJniWrapper create() { - return new VeloxBloomFilterJniWrapper(Runtimes.contextInstance()); + public static VeloxBloomFilterJniWrapper create(Runtime runtime) { + return new VeloxBloomFilterJniWrapper(runtime); } @Override diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index ed69a5893c25..582bf997fba1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -110,8 +110,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { GenericExpressionTransformer(condFuncName, Seq(left), condExpr), right, left, - newExpr - ) + newExpr) } /** Transform Uuid to Substrait. */ @@ -488,8 +487,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { left, right, isSkewJoin, - projectList - ) + projectList) } override def genCartesianProductExecTransformer( left: SparkPlan, @@ -498,8 +496,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { CartesianProductExecTransformer( ColumnarCartesianProductBridge(left), ColumnarCartesianProductBridge(right), - condition - ) + condition) } override def genBroadcastNestedLoopJoinExecTransformer( @@ -508,13 +505,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { buildSide: BuildSide, joinType: JoinType, condition: Option[Expression]): BroadcastNestedLoopJoinExecTransformer = - VeloxBroadcastNestedLoopJoinExecTransformer( - left, - right, - buildSide, - joinType, - condition - ) + VeloxBroadcastNestedLoopJoinExecTransformer(left, right, buildSide, joinType, condition) override def genHashExpressionTransformer( substraitExprName: String, @@ -795,10 +786,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { * * @return */ - override def genExtendedOptimizers(): List[SparkSession => Rule[LogicalPlan]] = List( - CollectRewriteRule.apply, - HLLRewriteRule.apply - ) + override def genExtendedOptimizers(): List[SparkSession => Rule[LogicalPlan]] = + List(CollectRewriteRule.apply, HLLRewriteRule.apply) /** * Generate extended columnar pre-rules, in the validation phase. @@ -879,8 +868,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { requiredChildOutput: Seq[Attribute], outer: Boolean, generatorOutput: Seq[Attribute], - child: SparkPlan - ): GenerateExecTransformerBase = { + child: SparkPlan): GenerateExecTransformerBase = { GenerateExecTransformer(generator, requiredChildOutput, outer, generatorOutput, child) } diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index ac24b53af11b..4cbde635e9bd 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -17,6 +17,7 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.backendsapi.TransformerApi +import org.apache.gluten.exec.Runtimes import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.gluten.utils.InputPartitionsUtil @@ -83,7 +84,8 @@ class VeloxTransformerApi extends TransformerApi with Logging { override def getNativePlanString(substraitPlan: Array[Byte], details: Boolean): String = { TaskResources.runUnsafe { - val jniWrapper = PlanEvaluatorJniWrapper.create() + val jniWrapper = PlanEvaluatorJniWrapper.create( + Runtimes.contextInstance("VeloxTransformerApi#getNativePlanString")) jniWrapper.nativePlanString(substraitPlan, details) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index d694f15fa9bd..289df1a6e54d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -21,7 +21,6 @@ import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenException import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized._ @@ -71,8 +70,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBas numInputRows, numOutputBatches, convertTime, - numRows - ) + numRows) } } @@ -97,9 +95,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBas numInputRows, numOutputBatches, convertTime, - numRows - ) - ) + numRows)) } // For spark 3.2. @@ -121,16 +117,16 @@ object RowToVeloxColumnarExec { val arrowSchema = SparkArrowUtil.toArrowSchema(schema, SQLConf.get.sessionLocalTimeZone) - val jniWrapper = NativeRowToColumnarJniWrapper.create() + val runtime = Runtimes.contextInstance("RowToColumnar") + val jniWrapper = NativeRowToColumnarJniWrapper.create(runtime) val arrowAllocator = ArrowBufferAllocators.contextInstance() - val memoryManager = NativeMemoryManagers.contextInstance("RowToColumnar") val cSchema = ArrowSchema.allocateNew(arrowAllocator) val factory = UnsafeProjection val converter = factory.create(schema) val r2cHandle = try { ArrowAbiUtil.exportSchema(arrowAllocator, arrowSchema, cSchema) - jniWrapper.init(cSchema.memoryAddress(), memoryManager.getNativeInstanceHandle) + jniWrapper.init(cSchema.memoryAddress()) } finally { cSchema.close() } @@ -220,7 +216,7 @@ object RowToVeloxColumnarExec { try { val handle = jniWrapper .nativeConvertRowToColumnar(r2cHandle, rowLength.toArray, arrowBuf.memoryAddress()) - val cb = ColumnarBatches.create(Runtimes.contextInstance(), handle) + val cb = ColumnarBatches.create(runtime, handle) convertTime += System.currentTimeMillis() - startNative cb } finally { diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala index 0d6714d3af92..1a54255208ea 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala @@ -18,8 +18,8 @@ package org.apache.gluten.execution import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenNotSupportException +import org.apache.gluten.exec.Runtimes import org.apache.gluten.extension.ValidationResult -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.NativeColumnarToRowJniWrapper @@ -75,13 +75,8 @@ case class VeloxColumnarToRowExec(child: SparkPlan) extends ColumnarToRowExecBas val convertTime = longMetric("convertTime") child.executeColumnar().mapPartitions { it => - VeloxColumnarToRowExec.toRowIterator( - it, - output, - numOutputRows, - numInputBatches, - convertTime - ) + VeloxColumnarToRowExec + .toRowIterator(it, output, numOutputRows, numInputBatches, convertTime) } } @@ -96,13 +91,7 @@ case class VeloxColumnarToRowExec(child: SparkPlan) extends ColumnarToRowExecBas sparkContext, mode, relation, - VeloxColumnarToRowExec.toRowIterator( - _, - output, - numOutputRows, - numInputBatches, - convertTime - )) + VeloxColumnarToRowExec.toRowIterator(_, output, numOutputRows, numInputBatches, convertTime)) } protected def withNewChildInternal(newChild: SparkPlan): VeloxColumnarToRowExec = @@ -120,10 +109,10 @@ object VeloxColumnarToRowExec { return Iterator.empty } + val runtime = Runtimes.contextInstance("ColumnarToRow") // TODO:: pass the jni jniWrapper and arrowSchema and serializeSchema method by broadcast - val jniWrapper = NativeColumnarToRowJniWrapper.create() - val c2rId = jniWrapper.nativeColumnarToRowInit( - NativeMemoryManagers.contextInstance("ColumnarToRow").getNativeInstanceHandle) + val jniWrapper = NativeColumnarToRowJniWrapper.create(runtime) + val c2rId = jniWrapper.nativeColumnarToRowInit() val res: Iterator[Iterator[InternalRow]] = new Iterator[Iterator[InternalRow]] { @@ -159,7 +148,7 @@ object VeloxColumnarToRowExec { val beforeConvert = System.currentTimeMillis() val batchHandle = ColumnarBatches.getNativeHandle(batch) val info = - jniWrapper.nativeColumnarToRowConvert(batchHandle, c2rId) + jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle) convertTime += (System.currentTimeMillis() - beforeConvert) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala index 6150507b4baa..3d7725655bb1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala @@ -17,8 +17,8 @@ package org.apache.gluten.utils import org.apache.gluten.datasource.DatasourceJniWrapper +import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkSchemaUtil @@ -38,13 +38,12 @@ object DatasourceUtil { def readSchema(file: FileStatus): Option[StructType] = { val allocator = ArrowBufferAllocators.contextInstance() - val datasourceJniWrapper = DatasourceJniWrapper.create() + val runtime = Runtimes.contextInstance("VeloxWriter") + val datasourceJniWrapper = DatasourceJniWrapper.create(runtime) val dsHandle = datasourceJniWrapper.nativeInitDatasource( file.getPath.toString, -1, - NativeMemoryManagers.contextInstance("VeloxWriter").getNativeInstanceHandle, - new util.HashMap[String, String]() - ) + new util.HashMap[String, String]()) val cSchema = ArrowSchema.allocateNew(allocator) datasourceJniWrapper.inspectSchema(dsHandle, cSchema.memoryAddress()) try { diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala index 6346251b6bcd..3d532133a053 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.nmm.NativeMemoryManagers +import org.apache.gluten.exec.Runtimes import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.vectorized.{ColumnarBatchSerializeResult, ColumnarBatchSerializerJniWrapper} @@ -31,7 +31,6 @@ import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.TaskResources -import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer; // Utility methods to convert Vanilla broadcast relations from/to Velox broadcast relations. @@ -153,17 +152,12 @@ object BroadcastUtils { if (filtered.isEmpty) { return ColumnarBatchSerializeResult.EMPTY } - val batchRuntime = ColumnarBatches.getRuntime(filtered.toList.asJava) val handleArray = filtered.map(ColumnarBatches.getNativeHandle) val serializeResult = try { ColumnarBatchSerializerJniWrapper - .forRuntime(batchRuntime) - .serialize( - handleArray, - NativeMemoryManagers - .contextInstance("BroadcastRelation") - .getNativeInstanceHandle) + .create(Runtimes.contextInstance("BroadcastUtils#serializeStream")) + .serialize(handleArray) } finally { filtered.foreach(ColumnarBatches.release) } diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index cb65b7504bfc..3f82f919b4d8 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -22,7 +22,6 @@ import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exec.Runtimes import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper @@ -150,8 +149,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe numInputRows, numOutputBatches, convertTime, - numRows - ) + numRows) } convertColumnarBatchToCachedBatch(rddColumnarBatch, schema, storageLevel, conf) } @@ -186,8 +184,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe selectedAttributes, numOutputRows, numInputBatches, - convertTime - ) + convertTime) } } @@ -198,10 +195,6 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe conf: SQLConf): RDD[CachedBatch] = { input.mapPartitions { it => - val nativeMemoryManagerHandle = NativeMemoryManagers - .contextInstance("ColumnarCachedBatchSerializer serialize") - .getNativeInstanceHandle - new Iterator[CachedBatch] { override def hasNext: Boolean = it.hasNext @@ -209,11 +202,8 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe val batch = it.next() val results = ColumnarBatchSerializerJniWrapper - .create() - .serialize( - Array(ColumnarBatches.getNativeHandle(batch)), - nativeMemoryManagerHandle - ) + .create(Runtimes.contextInstance("ColumnarCachedBatchSerializer#serialize")) + .serialize(Array(ColumnarBatches.getNativeHandle(batch))) CachedColumnarBatch( results.getNumRows.toInt, results.getSerialized.length, @@ -237,19 +227,15 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe val timezoneId = SQLConf.get.sessionLocalTimeZone input.mapPartitions { it => + val runtime = Runtimes.contextInstance("ColumnarCachedBatchSerializer#read") val jniWrapper = ColumnarBatchSerializerJniWrapper - .create() - val nmm = NativeMemoryManagers - .contextInstance("ColumnarCachedBatchSerializer read") + .create(runtime) val schema = SparkArrowUtil.toArrowSchema(localSchema, timezoneId) val arrowAlloc = ArrowBufferAllocators.contextInstance() val cSchema = ArrowSchema.allocateNew(arrowAlloc) ArrowAbiUtil.exportSchema(arrowAlloc, schema, cSchema) val deserializerHandle = jniWrapper - .init( - cSchema.memoryAddress(), - nmm.getNativeInstanceHandle - ) + .init(cSchema.memoryAddress()) cSchema.close() Iterators @@ -261,10 +247,10 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe val batchHandle = jniWrapper .deserialize(deserializerHandle, cachedBatch.bytes) - val batch = ColumnarBatches.create(Runtimes.contextInstance(), batchHandle) + val batch = ColumnarBatches.create(runtime, batchHandle) if (shouldSelectAttributes) { try { - ColumnarBatches.select(nmm, batch, requestedColumnIndices.toArray) + ColumnarBatches.select(batch, requestedColumnIndices.toArray) } finally { batch.close() } diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java index 4d887e60794a..56df7b9ad57f 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java @@ -53,7 +53,8 @@ public BlockStripe next() { return new BlockStripe() { @Override public ColumnarBatch getColumnarBatch() { - return ColumnarBatches.create(Runtimes.contextInstance(), blockAddresses[0]); + return ColumnarBatches.create( + Runtimes.contextInstance("VeloxBlockStripes"), blockAddresses[0]); } @Override diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala index 9f9d4332640c..ebbf959d0b1f 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala @@ -19,9 +19,9 @@ package org.apache.spark.sql.execution.datasources.velox import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.datasource.DatasourceJniWrapper import org.apache.gluten.exception.GlutenException +import org.apache.gluten.exec.Runtimes import org.apache.gluten.execution.datasource.GlutenRowSplitter import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.{ArrowAbiUtil, DatasourceUtil} import org.apache.spark.sql.SparkSession @@ -59,15 +59,13 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase { SparkArrowUtil.toArrowSchema(dataSchema, SQLConf.get.sessionLocalTimeZone) val cSchema = ArrowSchema.allocateNew(ArrowBufferAllocators.contextInstance()) var dsHandle = -1L - val datasourceJniWrapper = DatasourceJniWrapper.create() + val runtime = Runtimes.contextInstance("VeloxWriter") + val datasourceJniWrapper = DatasourceJniWrapper.create(runtime) val allocator = ArrowBufferAllocators.contextInstance() try { ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema) - dsHandle = datasourceJniWrapper.nativeInitDatasource( - filePath, - cSchema.memoryAddress(), - NativeMemoryManagers.contextInstance("VeloxWriter").getNativeInstanceHandle, - nativeConf) + dsHandle = + datasourceJniWrapper.nativeInitDatasource(filePath, cSchema.memoryAddress(), nativeConf) } catch { case e: IOException => throw new GlutenException(e) @@ -119,16 +117,12 @@ class VeloxRowSplitter extends GlutenRowSplitter { hasBucket: Boolean, reserve_partition_columns: Boolean = false): BlockStripes = { val handler = ColumnarBatches.getNativeHandle(row.batch) - val datasourceJniWrapper = DatasourceJniWrapper.create() + val runtime = Runtimes.contextInstance("VeloxPartitionWriter") + val datasourceJniWrapper = DatasourceJniWrapper.create(runtime) val originalColumns: Array[Int] = Array.range(0, row.batch.numCols()) val dataColIndice = originalColumns.filterNot(partitionColIndice.contains(_)) new VeloxBlockStripes( datasourceJniWrapper - .splitBlockByPartitionAndBucket( - handler, - dataColIndice, - hasBucket, - NativeMemoryManagers.contextInstance("VeloxPartitionWriter").getNativeInstanceHandle) - ) + .splitBlockByPartitionAndBucket(handler, dataColIndice, hasBucket)) } } diff --git a/cpp/core/compute/Runtime.cc b/cpp/core/compute/Runtime.cc index 387da1a0a897..c6bae1e7bc0b 100644 --- a/cpp/core/compute/Runtime.cc +++ b/cpp/core/compute/Runtime.cc @@ -56,9 +56,12 @@ void Runtime::registerFactory(const std::string& kind, Runtime::Factory factory) runtimeFactories().registerFactory(kind, std::move(factory)); } -Runtime* Runtime::create(const std::string& kind, const std::unordered_map& sessionConf) { +Runtime* Runtime::create( + const std::string& kind, + std::unique_ptr listener, + const std::unordered_map& sessionConf) { auto& factory = runtimeFactories().getFactory(kind); - return factory(sessionConf); + return factory(std::move(listener), sessionConf); } void Runtime::release(Runtime* runtime) { diff --git a/cpp/core/compute/Runtime.h b/cpp/core/compute/Runtime.h index 7574b0219dcf..fb501dc9acca 100644 --- a/cpp/core/compute/Runtime.h +++ b/cpp/core/compute/Runtime.h @@ -55,13 +55,18 @@ struct SparkTaskInfo { class Runtime : public std::enable_shared_from_this { public: - using Factory = std::function&)>; + using Factory = std::function< + Runtime*(std::unique_ptr listener, const std::unordered_map&)>; static void registerFactory(const std::string& kind, Factory factory); - static Runtime* create(const std::string& kind, const std::unordered_map& sessionConf = {}); + static Runtime* create( + const std::string& kind, + std::unique_ptr listener, + const std::unordered_map& sessionConf = {}); static void release(Runtime*); - Runtime() = default; - Runtime(const std::unordered_map& confMap) : confMap_(confMap) {} + Runtime(std::shared_ptr memoryManager, const std::unordered_map& confMap) + : memoryManager_(memoryManager), confMap_(confMap) {} + virtual ~Runtime() = default; virtual void parsePlan(const uint8_t* data, int32_t size, std::optional dumpFile) = 0; @@ -78,52 +83,40 @@ class Runtime : public std::enable_shared_from_this { } virtual std::shared_ptr createResultIterator( - MemoryManager* memoryManager, const std::string& spillDir, const std::vector>& inputs, const std::unordered_map& sessionConf) = 0; virtual std::shared_ptr createOrGetEmptySchemaBatch(int32_t numRows) = 0; - virtual std::shared_ptr - select(MemoryManager*, std::shared_ptr, std::vector) = 0; + virtual std::shared_ptr select(std::shared_ptr, std::vector) = 0; - virtual MemoryManager* createMemoryManager( - const std::string& name, - std::shared_ptr, - std::unique_ptr) = 0; + virtual MemoryManager* memoryManager() { + return memoryManager_.get(); + }; /// This function is used to create certain converter from the format used by /// the backend to Spark unsafe row. - virtual std::shared_ptr createColumnar2RowConverter(MemoryManager* memoryManager) = 0; + virtual std::shared_ptr createColumnar2RowConverter() = 0; - virtual std::shared_ptr createRow2ColumnarConverter( - MemoryManager* memoryManager, - struct ArrowSchema* cSchema) = 0; + virtual std::shared_ptr createRow2ColumnarConverter(struct ArrowSchema* cSchema) = 0; virtual std::shared_ptr createShuffleWriter( int numPartitions, std::unique_ptr partitionWriter, - ShuffleWriterOptions options, - MemoryManager* memoryManager) = 0; + ShuffleWriterOptions options) = 0; virtual Metrics* getMetrics(ColumnarBatchIterator* rawIter, int64_t exportNanos) = 0; virtual std::shared_ptr createDatasource( const std::string& filePath, - MemoryManager* memoryManager, std::shared_ptr schema) = 0; virtual std::shared_ptr createShuffleReader( std::shared_ptr schema, - ShuffleReaderOptions options, - arrow::MemoryPool* pool, - MemoryManager* memoryManager) = 0; + ShuffleReaderOptions options) = 0; - virtual std::unique_ptr createColumnarBatchSerializer( - MemoryManager* memoryManager, - arrow::MemoryPool* arrowPool, - struct ArrowSchema* cSchema) = 0; + virtual std::unique_ptr createColumnarBatchSerializer(struct ArrowSchema* cSchema) = 0; virtual void dumpConf(const std::string& path) = 0; @@ -135,17 +128,18 @@ class Runtime : public std::enable_shared_from_this { taskInfo_ = taskInfo; } - ObjectStore* objectStore() { - return objStore_.get(); + ObjectHandle saveObject(std::shared_ptr obj) { + return objStore_->save(obj); } protected: + std::shared_ptr memoryManager_; std::unique_ptr objStore_ = ObjectStore::create(); + std::unordered_map confMap_; // Session conf map + ::substrait::Plan substraitPlan_; std::vector<::substrait::ReadRel_LocalFiles> localFiles_; std::optional writeFilesTempPath_; SparkTaskInfo taskInfo_; - // Session conf map - std::unordered_map confMap_; }; } // namespace gluten diff --git a/cpp/core/jni/JniCommon.cc b/cpp/core/jni/JniCommon.cc index 08c5cb1d40cb..759a9d121f91 100644 --- a/cpp/core/jni/JniCommon.cc +++ b/cpp/core/jni/JniCommon.cc @@ -112,7 +112,7 @@ std::shared_ptr gluten::JniColumnarBatchIterator::next() checkException(env); jlong handle = env->CallLongMethod(jColumnarBatchItr_, serializedColumnarBatchIteratorNext_); checkException(env); - auto batch = runtime_->objectStore()->retrieve(handle); + auto batch = ObjectStore::retrieve(handle); if (writer_ != nullptr) { // save snapshot of the batch to file std::shared_ptr schema = batch->exportArrowSchema(); diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index bc5cf84f6ff4..d5c9f2b3b18b 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -28,6 +28,7 @@ #include "memory/AllocationListener.h" #include "shuffle/rss/RssClient.h" #include "utils/Compression.h" +#include "utils/ObjectStore.h" #include "utils/ResourceMap.h" #include "utils/exception.h" @@ -121,7 +122,7 @@ static inline void attachCurrentThreadAsDaemonOrThrow(JavaVM* vm, JNIEnv** out) } template -static T* jniCastOrThrow(gluten::ResourceHandle handle) { +static T* jniCastOrThrow(jlong handle) { auto instance = reinterpret_cast(handle); GLUTEN_CHECK(instance != nullptr, "FATAL: resource instance should not be null."); return instance; diff --git a/cpp/core/jni/JniError.cc b/cpp/core/jni/JniError.cc index af21ba2a1f3e..b0f4e713a948 100644 --- a/cpp/core/jni/JniError.cc +++ b/cpp/core/jni/JniError.cc @@ -41,7 +41,13 @@ jclass gluten::JniErrorState::illegalAccessExceptionClass() { return illegalAccessExceptionClass_; } +jclass gluten::JniErrorState::glutenExceptionClass() { + assertInitialized(); + return glutenExceptionClass_; +} + void gluten::JniErrorState::initialize(JNIEnv* env) { + glutenExceptionClass_ = createGlobalClassReference(env, "Lorg/apache/gluten/exception/GlutenException;"); ioExceptionClass_ = createGlobalClassReference(env, "Ljava/io/IOException;"); runtimeExceptionClass_ = createGlobalClassReference(env, "Ljava/lang/RuntimeException;"); unsupportedOperationExceptionClass_ = createGlobalClassReference(env, "Ljava/lang/UnsupportedOperationException;"); @@ -61,6 +67,7 @@ void gluten::JniErrorState::close() { } JNIEnv* env; attachCurrentThreadAsDaemonOrThrow(vm_, &env); + env->DeleteGlobalRef(glutenExceptionClass_); env->DeleteGlobalRef(ioExceptionClass_); env->DeleteGlobalRef(runtimeExceptionClass_); env->DeleteGlobalRef(unsupportedOperationExceptionClass_); diff --git a/cpp/core/jni/JniError.h b/cpp/core/jni/JniError.h index f902ebe0367e..828c90861292 100644 --- a/cpp/core/jni/JniError.h +++ b/cpp/core/jni/JniError.h @@ -28,11 +28,11 @@ #endif #ifndef JNI_METHOD_END -#define JNI_METHOD_END(fallback_expr) \ - } \ - catch (std::exception & e) { \ - env->ThrowNew(gluten::getJniErrorState()->runtimeExceptionClass(), e.what()); \ - return fallback_expr; \ +#define JNI_METHOD_END(fallback_expr) \ + } \ + catch (std::exception & e) { \ + env->ThrowNew(gluten::getJniErrorState()->glutenExceptionClass(), e.what()); \ + return fallback_expr; \ } // macro ended #endif @@ -53,6 +53,8 @@ class JniErrorState { jclass illegalAccessExceptionClass(); + jclass glutenExceptionClass(); + private: void initialize(JNIEnv* env); @@ -61,6 +63,7 @@ class JniErrorState { jclass unsupportedOperationExceptionClass_ = nullptr; jclass illegalAccessExceptionClass_ = nullptr; jclass illegalArgumentExceptionClass_ = nullptr; + jclass glutenExceptionClass_ = nullptr; JavaVM* vm_; bool initialized_{false}; bool closed_{false}; diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 1e5326689229..23eea2db7ce6 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -180,7 +180,7 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { javaReservationListenerClass = createGlobalClassReference( env, - "Lorg/apache/gluten/memory/nmm/" + "Lorg/apache/gluten/memory/listener/" "ReservationListener;"); reserveMemoryMethod = getMethodIdOrError(env, javaReservationListenerClass, "reserve", "(J)J"); @@ -219,14 +219,63 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRunt JNIEnv* env, jclass, jstring jbackendType, + jobject jlistener, jbyteArray sessionConf) { JNI_METHOD_START + JavaVM* vm; + if (env->GetJavaVM(&vm) != JNI_OK) { + throw gluten::GlutenException("Unable to get JavaVM instance"); + } + auto backendType = jStringToCString(env, jbackendType); + std::unique_ptr listener = + std::make_unique(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod); + auto safeArray = gluten::getByteArrayElementsSafe(env, sessionConf); auto sparkConf = gluten::parseConfMap(env, safeArray.elems(), safeArray.length()); - auto runtime = gluten::Runtime::create(backendType, sparkConf); + auto runtime = gluten::Runtime::create(backendType, std::move(listener), sparkConf); return reinterpret_cast(runtime); - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) +} + +JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_collectMemoryUsage( // NOLINT + JNIEnv* env, + jclass, + jlong ctxHandle) { + JNI_METHOD_START + auto runtime = jniCastOrThrow(ctxHandle); + + const MemoryUsageStats& stats = runtime->memoryManager()->collectMemoryUsageStats(); + auto size = stats.ByteSizeLong(); + jbyteArray out = env->NewByteArray(size); + uint8_t buffer[size]; + GLUTEN_CHECK( + stats.SerializeToArray(reinterpret_cast(buffer), size), + "Serialization failed when collecting memory usage stats"); + env->SetByteArrayRegion(out, 0, size, reinterpret_cast(buffer)); + return out; + JNI_METHOD_END(nullptr) +} + +JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_shrinkMemory( // NOLINT + JNIEnv* env, + jclass, + jlong ctxHandle, + jlong size) { + JNI_METHOD_START + auto runtime = jniCastOrThrow(ctxHandle); + return runtime->memoryManager()->shrink(static_cast(size)); + JNI_METHOD_END(kInvalidObjectHandle) +} + +JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_holdMemory( // NOLINT + JNIEnv* env, + jclass, + jlong ctxHandle) { + JNI_METHOD_START + auto runtime = jniCastOrThrow(ctxHandle); + runtime->memoryManager()->hold(); + JNI_METHOD_END() } JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_releaseRuntime( // NOLINT @@ -278,7 +327,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWithIterator( // NOLINT JNIEnv* env, jobject wrapper, - jlong memoryManagerHandle, jbyteArray planArr, jobjectArray splitInfosArr, jobjectArray iterArr, @@ -308,8 +356,6 @@ Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWith ctx->dumpConf(saveDir + "/conf" + fileIdentifier + ".ini"); } - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - auto spillDirStr = jStringToCString(env, spillDir); auto safePlanArray = gluten::getByteArrayElementsSafe(env, planArr); @@ -346,8 +392,8 @@ Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWith inputIters.push_back(std::move(resultIter)); } - return ctx->objectStore()->save(ctx->createResultIterator(memoryManager, spillDirStr, inputIters, conf)); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->createResultIterator(spillDirStr, inputIters, conf)); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jboolean JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeHasNext( // NOLINT @@ -357,7 +403,7 @@ JNIEXPORT jboolean JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIte JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto iter = ctx->objectStore()->retrieve(iterHandle); + auto iter = ObjectStore::retrieve(iterHandle); if (iter == nullptr) { std::string errorMessage = "When hasNext() is called on a closed iterator, an exception is thrown. To prevent this, consider using the protectInvocationFlow() method when creating the iterator in scala side. This will allow the hasNext() method to be called multiple times without issue."; @@ -374,17 +420,17 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterat JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto iter = ctx->objectStore()->retrieve(iterHandle); + auto iter = ObjectStore::retrieve(iterHandle); if (!iter->hasNext()) { - return kInvalidResourceHandle; + return kInvalidObjectHandle; } std::shared_ptr batch = iter->next(); - auto batchHandle = ctx->objectStore()->save(batch); + auto batchHandle = ctx->saveObject(batch); iter->setExportNanos(batch->getExportNanos()); return batchHandle; - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeFetchMetrics( // NOLINT @@ -394,7 +440,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIter JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto iter = ctx->objectStore()->retrieve(iterHandle); + auto iter = ObjectStore::retrieve(iterHandle); auto metrics = iter->getMetrics(); unsigned int numMetrics = 0; if (metrics) { @@ -455,13 +501,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterat JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto it = ctx->objectStore()->retrieve(iterHandle); + auto it = ObjectStore::retrieve(iterHandle); if (it == nullptr) { std::string errorMessage = "Invalid result iter handle " + std::to_string(iterHandle); throw gluten::GlutenException(errorMessage); } return it->spillFixedSize(size); - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeClose( // NOLINT @@ -471,34 +517,32 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterato JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - ctx->objectStore()->release(iterHandle); + ObjectStore::release(iterHandle); JNI_METHOD_END() } JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_NativeColumnarToRowJniWrapper_nativeColumnarToRowInit( // NOLINT JNIEnv* env, - jobject wrapper, - jlong memoryManagerHandle) { + jobject wrapper) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); // Convert the native batch to Spark unsafe row. - return ctx->objectStore()->save(ctx->createColumnar2RowConverter(memoryManager)); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->createColumnar2RowConverter()); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_NativeColumnarToRowJniWrapper_nativeColumnarToRowConvert( // NOLINT JNIEnv* env, jobject wrapper, - jlong batchHandle, - jlong c2rHandle) { + jlong c2rHandle, + jlong batchHandle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto columnarToRowConverter = ctx->objectStore()->retrieve(c2rHandle); - auto cb = ctx->objectStore()->retrieve(batchHandle); + auto columnarToRowConverter = ObjectStore::retrieve(c2rHandle); + auto cb = ObjectStore::retrieve(batchHandle); columnarToRowConverter->convert(cb); const auto& offsets = columnarToRowConverter->getOffsets(); @@ -527,22 +571,19 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_NativeColumnarToRowJniW JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - ctx->objectStore()->release(c2rHandle); + ObjectStore::release(c2rHandle); JNI_METHOD_END() } JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_NativeRowToColumnarJniWrapper_init( // NOLINT JNIEnv* env, jobject wrapper, - jlong cSchema, - jlong memoryManagerHandle) { + jlong cSchema) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - return ctx->objectStore()->save( - ctx->createRow2ColumnarConverter(memoryManager, reinterpret_cast(cSchema))); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->createRow2ColumnarConverter(reinterpret_cast(cSchema))); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL @@ -562,10 +603,10 @@ Java_org_apache_gluten_vectorized_NativeRowToColumnarJniWrapper_nativeConvertRow auto safeArray = gluten::getLongArrayElementsSafe(env, rowLength); uint8_t* address = reinterpret_cast(memoryAddress); - auto converter = ctx->objectStore()->retrieve(r2cHandle); + auto converter = ObjectStore::retrieve(r2cHandle); auto cb = converter->convert(numRows, safeArray.elems(), address); - return ctx->objectStore()->save(cb); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(cb); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_NativeRowToColumnarJniWrapper_close( // NOLINT @@ -575,7 +616,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_NativeRowToColumnarJniW JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - ctx->objectStore()->release(r2cHandle); + ObjectStore::release(r2cHandle); JNI_METHOD_END() } @@ -585,7 +626,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniW jlong batchHandle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); return env->NewStringUTF(batch->getType().c_str()); JNI_METHOD_END(nullptr) } @@ -596,9 +637,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra jlong batchHandle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); return batch->numBytes(); - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_numColumns( // NOLINT @@ -607,9 +648,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra jlong batchHandle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); return batch->numColumns(); - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_numRows( // NOLINT @@ -618,9 +659,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra jlong batchHandle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); return batch->numRows(); - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_compose( // NOLINT @@ -636,12 +677,12 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra std::vector> batches; for (int i = 0; i < handleCount; ++i) { int64_t handle = safeArray.elems()[i]; - auto batch = ctx->objectStore()->retrieve(handle); + auto batch = ObjectStore::retrieve(handle); batches.push_back(batch); } auto newBatch = CompositeColumnarBatch::create(std::move(batches)); - return ctx->objectStore()->save(newBatch); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(newBatch); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_exportToArrow( // NOLINT @@ -652,7 +693,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrap jlong cArray) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); std::shared_ptr exportedSchema = batch->exportArrowSchema(); std::shared_ptr exportedArray = batch->exportArrowArray(); ArrowSchemaMove(exportedSchema.get(), reinterpret_cast(cSchema)); @@ -676,8 +717,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra ArrowSchemaMove(arrowSchema, targetSchema.get()); std::shared_ptr batch = std::make_shared(std::move(targetSchema), std::move(targetArray)); - return ctx->objectStore()->save(batch); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(batch); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_getForEmptySchema( // NOLINT @@ -686,19 +727,17 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra jint numRows) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - return ctx->objectStore()->save(ctx->createOrGetEmptySchemaBatch(static_cast(numRows))); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->createOrGetEmptySchemaBatch(static_cast(numRows))); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_select( // NOLINT JNIEnv* env, jobject wrapper, - jlong memoryManagerHandle, jlong batchHandle, jintArray jcolumnIndices) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); auto safeArray = gluten::getIntArrayElementsSafe(env, jcolumnIndices); int size = env->GetArrayLength(jcolumnIndices); @@ -707,9 +746,20 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra columnIndices.push_back(safeArray.elems()[i]); } - return ctx->objectStore()->save( - ctx->select(memoryManager, ctx->objectStore()->retrieve(batchHandle), std::move(columnIndices))); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->select(ObjectStore::retrieve(batchHandle), std::move(columnIndices))); + JNI_METHOD_END(kInvalidObjectHandle) +} + +JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_obtainOwnership( // NOLINT + JNIEnv* env, + jobject wrapper, + jlong batchHandle) { + JNI_METHOD_START + auto ctx = gluten::getRuntime(env, wrapper); + auto batch = ObjectStore::retrieve(batchHandle); + auto newHandle = ctx->saveObject(batch); + return newHandle; + JNI_METHOD_END(-1L) } JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_close( // NOLINT @@ -718,7 +768,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrap jlong batchHandle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - ctx->objectStore()->release(batchHandle); + ObjectStore::release(batchHandle); JNI_METHOD_END() } @@ -739,7 +789,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe jstring dataFileJstr, jint numSubDirs, jstring localDirsJstr, - jlong memoryManagerHandle, jdouble reallocThreshold, jlong firstBatchHandle, jlong taskAttemptId, @@ -751,7 +800,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe jstring shuffleWriterTypeJstr) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); if (partitioningNameJstr == nullptr) { throw gluten::GlutenException(std::string("Short partitioning name can't be null")); } @@ -813,7 +861,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe partitionWriter = std::make_unique( numPartitions, std::move(partitionWriterOptions), - memoryManager->getArrowMemoryPool(), + ctx->memoryManager()->getArrowMemoryPool(), dataFile, configuredDirs); } else if (partitionWriterType == "celeborn") { @@ -830,7 +878,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe partitionWriter = std::make_unique( numPartitions, std::move(partitionWriterOptions), - memoryManager->getArrowMemoryPool(), + ctx->memoryManager()->getArrowMemoryPool(), std::move(celebornClient)); } else if (partitionWriterType == "uniffle") { jclass unifflePartitionPusherClass = @@ -846,15 +894,15 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe partitionWriter = std::make_unique( numPartitions, std::move(partitionWriterOptions), - memoryManager->getArrowMemoryPool(), + ctx->memoryManager()->getArrowMemoryPool(), std::move(uniffleClient)); } else { throw gluten::GlutenException("Unrecognizable partition writer type: " + partitionWriterType); } - return ctx->objectStore()->save(ctx->createShuffleWriter( - numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions), memoryManager)); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject( + ctx->createShuffleWriter(numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions))); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_nativeEvict( // NOLINT @@ -866,7 +914,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto shuffleWriter = ctx->objectStore()->retrieve(shuffleWriterHandle); + auto shuffleWriter = ObjectStore::retrieve(shuffleWriterHandle); if (!shuffleWriter) { std::string errorMessage = "Invalid shuffle writer handle " + std::to_string(shuffleWriterHandle); throw gluten::GlutenException(errorMessage); @@ -875,7 +923,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe gluten::arrowAssertOkOrThrow( shuffleWriter->reclaimFixedSize(size, &evictedSize), "(shuffle) nativeEvict: evict failed"); return (jlong)evictedSize; - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_write( // NOLINT @@ -888,18 +936,18 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto shuffleWriter = ctx->objectStore()->retrieve(shuffleWriterHandle); + auto shuffleWriter = ObjectStore::retrieve(shuffleWriterHandle); if (!shuffleWriter) { std::string errorMessage = "Invalid shuffle writer handle " + std::to_string(shuffleWriterHandle); throw gluten::GlutenException(errorMessage); } // The column batch maybe VeloxColumnBatch or ArrowCStructColumnarBatch(FallbackRangeShuffleWriter) - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); auto numBytes = batch->numBytes(); gluten::arrowAssertOkOrThrow(shuffleWriter->write(batch, memLimit), "Native write: shuffle writer failed"); return numBytes; - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_stop( // NOLINT @@ -909,7 +957,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrap JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto shuffleWriter = ctx->objectStore()->retrieve(shuffleWriterHandle); + auto shuffleWriter = ObjectStore::retrieve(shuffleWriterHandle); if (!shuffleWriter) { std::string errorMessage = "Invalid shuffle writer handle " + std::to_string(shuffleWriterHandle); throw gluten::GlutenException(errorMessage); @@ -951,7 +999,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - ctx->objectStore()->release(shuffleWriterHandle); + ObjectStore::release(shuffleWriterHandle); JNI_METHOD_END() } @@ -971,16 +1019,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe JNIEnv* env, jobject wrapper, jlong cSchema, - jlong memoryManagerHandle, jstring compressionType, jstring compressionBackend, jint batchSize, jstring shuffleWriterType) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - auto pool = memoryManager->getArrowMemoryPool(); ShuffleReaderOptions options = ShuffleReaderOptions{}; options.compressionType = getCompressionType(env, compressionType); options.compressionTypeStr = getCompressionTypeStr(env, compressionType); @@ -996,8 +1041,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe std::shared_ptr schema = gluten::arrowGetOrThrow(arrow::ImportSchema(reinterpret_cast(cSchema))); - return ctx->objectStore()->save(ctx->createShuffleReader(schema, options, pool, memoryManager)); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->createShuffleReader(schema, options)); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper_readStream( // NOLINT @@ -1007,11 +1052,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe jobject jniIn) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto reader = ctx->objectStore()->retrieve(shuffleReaderHandle); + auto reader = ObjectStore::retrieve(shuffleReaderHandle); std::shared_ptr in = std::make_shared(env, reader->getPool(), jniIn); auto outItr = reader->readStream(in); - return ctx->objectStore()->save(outItr); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(outItr); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper_populateMetrics( // NOLINT @@ -1022,7 +1067,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto reader = ctx->objectStore()->retrieve(shuffleReaderHandle); + auto reader = ObjectStore::retrieve(shuffleReaderHandle); env->CallVoidMethod(metrics, shuffleReaderMetricsSetDecompressTime, reader->getDecompressTime()); env->CallVoidMethod(metrics, shuffleReaderMetricsSetDeserializeTime, reader->getDeserializeTime()); @@ -1037,9 +1082,9 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto reader = ctx->objectStore()->retrieve(shuffleReaderHandle); + auto reader = ObjectStore::retrieve(shuffleReaderHandle); GLUTEN_THROW_NOT_OK(reader->close()); - ctx->objectStore()->release(shuffleReaderHandle); + ObjectStore::release(shuffleReaderHandle); JNI_METHOD_END() } @@ -1048,30 +1093,28 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_n jobject wrapper, jstring filePath, jlong cSchema, - jlong memoryManagerHandle, jbyteArray options) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - ResourceHandle handle = kInvalidResourceHandle; + ObjectHandle handle = kInvalidObjectHandle; if (cSchema == -1) { // Only inspect the schema and not write - handle = ctx->objectStore()->save(ctx->createDatasource(jStringToCString(env, filePath), memoryManager, nullptr)); + handle = ctx->saveObject(ctx->createDatasource(jStringToCString(env, filePath), nullptr)); } else { auto safeArray = gluten::getByteArrayElementsSafe(env, options); auto datasourceOptions = gluten::parseConfMap(env, safeArray.elems(), safeArray.length()); auto& sparkConf = ctx->getConfMap(); datasourceOptions.insert(sparkConf.begin(), sparkConf.end()); auto schema = gluten::arrowGetOrThrow(arrow::ImportSchema(reinterpret_cast(cSchema))); - handle = ctx->objectStore()->save(ctx->createDatasource(jStringToCString(env, filePath), memoryManager, schema)); - auto datasource = ctx->objectStore()->retrieve(handle); + handle = ctx->saveObject(ctx->createDatasource(jStringToCString(env, filePath), schema)); + auto datasource = ObjectStore::retrieve(handle); datasource->init(datasourceOptions); } return handle; - JNI_METHOD_END(kInvalidResourceHandle) + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_inspectSchema( // NOLINT @@ -1082,7 +1125,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_in JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto datasource = ctx->objectStore()->retrieve(dsHandle); + auto datasource = ObjectStore::retrieve(dsHandle); datasource->inspectSchema(reinterpret_cast(cSchema)); JNI_METHOD_END() } @@ -1094,9 +1137,9 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_cl JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto datasource = ctx->objectStore()->retrieve(dsHandle); + auto datasource = ObjectStore::retrieve(dsHandle); datasource->close(); - ctx->objectStore()->release(dsHandle); + ObjectStore::release(dsHandle); JNI_METHOD_END() } @@ -1107,7 +1150,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_wr jobject jIter) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto datasource = ctx->objectStore()->retrieve(dsHandle); + auto datasource = ObjectStore::retrieve(dsHandle); auto iter = makeJniColumnarBatchIterator(env, jIter, ctx, nullptr); while (true) { auto batch = iter->next(); @@ -1129,7 +1172,7 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB jlong memoryManagerId) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ctx->objectStore()->retrieve(batchHandle); + auto batch = ObjectStore::retrieve(batchHandle); auto safeArray = gluten::getIntArrayElementsSafe(env, partitionColIndice); int size = env->GetArrayLength(partitionColIndice); std::vector partitionColIndiceVec; @@ -1137,10 +1180,9 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB partitionColIndiceVec.push_back(safeArray.elems()[i]); } - MemoryManager* memoryManager = reinterpret_cast(memoryManagerId); auto result = batch->toUnsafeRow(0); auto rowBytes = result.data(); - auto newBatchHandle = ctx->objectStore()->save(ctx->select(memoryManager, batch, partitionColIndiceVec)); + auto newBatchHandle = ctx->saveObject(ctx->select(batch, partitionColIndiceVec)); auto bytesSize = result.size(); jbyteArray bytesArray = env->NewByteArray(bytesSize); @@ -1158,138 +1200,12 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB JNI_METHOD_END(nullptr) } -JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_alloc_NativeMemoryAllocator_getAllocator( // NOLINT - JNIEnv* env, - jclass, - jstring jTypeName) { - JNI_METHOD_START - std::string typeName = jStringToCString(env, jTypeName); - std::shared_ptr* allocator = new std::shared_ptr; - if (typeName == "DEFAULT") { - *allocator = defaultMemoryAllocator(); - } else { - delete allocator; - allocator = nullptr; - throw GlutenException("Unexpected allocator type name: " + typeName); - } - return reinterpret_cast(allocator); - JNI_METHOD_END(kInvalidResourceHandle) -} - -JNIEXPORT void JNICALL Java_org_apache_gluten_memory_alloc_NativeMemoryAllocator_releaseAllocator( // NOLINT - JNIEnv* env, - jclass, - jlong allocatorId) { - JNI_METHOD_START - delete reinterpret_cast*>(allocatorId); - JNI_METHOD_END() -} - -JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_alloc_NativeMemoryAllocator_bytesAllocated( // NOLINT - JNIEnv* env, - jclass, - jlong allocatorId) { - JNI_METHOD_START - auto* alloc = reinterpret_cast*>(allocatorId); - if (alloc == nullptr) { - throw gluten::GlutenException("Memory allocator instance not found. It may not exist nor has been closed"); - } - return (*alloc)->getBytes(); - JNI_METHOD_END(kInvalidResourceHandle) -} - -JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_create( // NOLINT - JNIEnv* env, - jclass, - jstring jbackendType, - jstring jnmmName, - jlong allocatorId, - jobject jlistener) { - JNI_METHOD_START - JavaVM* vm; - if (env->GetJavaVM(&vm) != JNI_OK) { - throw gluten::GlutenException("Unable to get JavaVM instance"); - } - auto allocator = reinterpret_cast*>(allocatorId); - if (allocator == nullptr) { - throw gluten::GlutenException("Allocator does not exist or has been closed"); - } - - std::unique_ptr listener = - std::make_unique(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod); - - if (gluten::backtrace_allocation) { - listener = std::make_unique(std::move(listener)); - } - - auto name = jStringToCString(env, jnmmName); - auto backendType = jStringToCString(env, jbackendType); - // TODO: move memory manager into Runtime then we can use more general Runtime. - auto runtime = gluten::Runtime::create(backendType); - auto manager = runtime->createMemoryManager(name, *allocator, std::move(listener)); - gluten::Runtime::release(runtime); - return reinterpret_cast(manager); - JNI_METHOD_END(kInvalidResourceHandle) -} - -JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_collectMemoryUsage( // NOLINT - JNIEnv* env, - jclass, - jlong memoryManagerHandle) { - JNI_METHOD_START - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - - const MemoryUsageStats& stats = memoryManager->collectMemoryUsageStats(); - auto size = stats.ByteSizeLong(); - jbyteArray out = env->NewByteArray(size); - uint8_t buffer[size]; - GLUTEN_CHECK( - stats.SerializeToArray(reinterpret_cast(buffer), size), - "Serialization failed when collecting memory usage stats"); - env->SetByteArrayRegion(out, 0, size, reinterpret_cast(buffer)); - return out; - JNI_METHOD_END(nullptr) -} - -JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_shrink( // NOLINT - JNIEnv* env, - jclass, - jlong memoryManagerHandle, - jlong size) { - JNI_METHOD_START - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - return memoryManager->shrink(static_cast(size)); - JNI_METHOD_END(kInvalidResourceHandle) -} - -JNIEXPORT void JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_hold( // NOLINT - JNIEnv* env, - jclass, - jlong memoryManagerHandle) { - JNI_METHOD_START - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - memoryManager->hold(); - JNI_METHOD_END() -} - -JNIEXPORT void JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_release( // NOLINT - JNIEnv* env, - jclass, - jlong memoryManagerHandle) { - JNI_METHOD_START - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - delete memoryManager; - JNI_METHOD_END() -} - JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_serialize( // NOLINT JNIEnv* env, jobject wrapper, - jlongArray handles, - jlong memoryManagerHandle) { + jlongArray handles) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); int32_t numBatches = env->GetArrayLength(handles); auto safeArray = gluten::getLongArrayElementsSafe(env, handles); @@ -1297,15 +1213,14 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSeriali std::vector> batches; int64_t numRows = 0L; for (int32_t i = 0; i < numBatches; i++) { - auto batch = ctx->objectStore()->retrieve(safeArray.elems()[i]); + auto batch = ObjectStore::retrieve(safeArray.elems()[i]); GLUTEN_DCHECK( batch != nullptr, "Cannot find the ColumnarBatch with handle " + std::to_string(safeArray.elems()[i])); numRows += batch->numRows(); batches.emplace_back(batch); } - auto arrowPool = memoryManager->getArrowMemoryPool(); - auto serializer = ctx->createColumnarBatchSerializer(memoryManager, arrowPool, nullptr); + auto serializer = ctx->createColumnarBatchSerializer(nullptr); auto buffer = serializer->serializeColumnarBatches(batches); auto bufferArr = env->NewByteArray(buffer->size()); env->SetByteArrayRegion(bufferArr, 0, buffer->size(), reinterpret_cast(buffer->data())); @@ -1320,16 +1235,11 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSeriali JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_init( // NOLINT JNIEnv* env, jobject wrapper, - jlong cSchema, - jlong memoryManagerHandle) { + jlong cSchema) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - - auto arrowPool = memoryManager->getArrowMemoryPool(); - return ctx->objectStore()->save( - ctx->createColumnarBatchSerializer(memoryManager, arrowPool, reinterpret_cast(cSchema))); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(ctx->createColumnarBatchSerializer(reinterpret_cast(cSchema))); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_deserialize( // NOLINT @@ -1340,13 +1250,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerialize JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto serializer = ctx->objectStore()->retrieve(serializerHandle); + auto serializer = ObjectStore::retrieve(serializerHandle); GLUTEN_DCHECK(serializer != nullptr, "ColumnarBatchSerializer cannot be null"); int32_t size = env->GetArrayLength(data); auto safeArray = gluten::getByteArrayElementsSafe(env, data); auto batch = serializer->deserialize(safeArray.elems(), size); - return ctx->objectStore()->save(batch); - JNI_METHOD_END(kInvalidResourceHandle) + return ctx->saveObject(batch); + JNI_METHOD_END(kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_close( // NOLINT @@ -1356,7 +1266,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializer JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - ctx->objectStore()->release(serializerHandle); + ObjectStore::release(serializerHandle); JNI_METHOD_END() } diff --git a/cpp/core/tests/CMakeLists.txt b/cpp/core/tests/CMakeLists.txt index d8efd9d40444..a0f3406f5c19 100644 --- a/cpp/core/tests/CMakeLists.txt +++ b/cpp/core/tests/CMakeLists.txt @@ -18,3 +18,4 @@ if(ENABLE_HBM) endif() add_test_case(round_robin_partitioner_test SOURCES RoundRobinPartitionerTest.cc) +add_test_case(objectstore__test SOURCES ObjectStoreTest.cc) diff --git a/cpp/core/tests/ObjectStoreTest.cc b/cpp/core/tests/ObjectStoreTest.cc new file mode 100644 index 000000000000..cb5ab3a87707 --- /dev/null +++ b/cpp/core/tests/ObjectStoreTest.cc @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/ObjectStore.h" +#include + +using namespace gluten; + +TEST(ObjectStore, retreive) { + auto store = ObjectStore::create(); + auto obj = std::make_shared(1); + auto handle = store->save(obj); + auto retrieved = ObjectStore::retrieve(handle); + ASSERT_EQ(*retrieved, 1); +} + +TEST(ObjectStore, retreiveMultiple) { + auto store = ObjectStore::create(); + auto obj1 = std::make_shared(50); + auto obj2 = std::make_shared(100); + auto handle1 = store->save(obj1); + auto handle2 = store->save(obj2); + auto retrieved1 = ObjectStore::retrieve(handle1); + auto retrieved2 = ObjectStore::retrieve(handle2); + ASSERT_EQ(*retrieved1, *obj1); + ASSERT_EQ(*retrieved2, *obj2); +} + +TEST(ObjectStore, release) { + ObjectHandle handle = kInvalidObjectHandle; + auto store = ObjectStore::create(); + { + auto obj = std::make_shared(1); + handle = store->save(obj); + } + auto retrieved = ObjectStore::retrieve(handle); + ASSERT_EQ(*retrieved, 1); + ObjectStore::release(handle); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle)); +} + +TEST(ObjectStore, releaseMultiple) { + ObjectHandle handle1 = kInvalidObjectHandle; + ObjectHandle handle2 = kInvalidObjectHandle; + auto store = ObjectStore::create(); + { + auto obj1 = std::make_shared(50); + auto obj2 = std::make_shared(100); + handle1 = store->save(obj1); + handle2 = store->save(obj2); + } + ASSERT_EQ(*ObjectStore::retrieve(handle1), 50); + ASSERT_EQ(*ObjectStore::retrieve(handle2), 100); + ObjectStore::release(handle2); + ASSERT_EQ(*ObjectStore::retrieve(handle1), 50); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle2)); + ObjectStore::release(handle1); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle1)); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle2)); +} + +TEST(ObjectStore, releaseObjectsInMultipleStores) { + ObjectHandle handle1 = kInvalidObjectHandle; + ObjectHandle handle2 = kInvalidObjectHandle; + auto store1 = ObjectStore::create(); + auto store2 = ObjectStore::create(); + { + auto obj1 = std::make_shared(50); + auto obj2 = std::make_shared(100); + handle1 = store1->save(obj1); + handle2 = store2->save(obj2); + } + ASSERT_EQ(*ObjectStore::retrieve(handle1), 50); + ASSERT_EQ(*ObjectStore::retrieve(handle2), 100); + ObjectStore::release(handle2); + ASSERT_EQ(*ObjectStore::retrieve(handle1), 50); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle2)); + ObjectStore::release(handle1); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle1)); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle2)); +} + +TEST(ObjectStore, releaseMultipleStores) { + ObjectHandle handle1 = kInvalidObjectHandle; + ObjectHandle handle2 = kInvalidObjectHandle; + auto store1 = ObjectStore::create(); + auto store2 = ObjectStore::create(); + { + auto obj1 = std::make_shared(50); + auto obj2 = std::make_shared(100); + handle1 = store1->save(obj1); + handle2 = store2->save(obj2); + } + ASSERT_EQ(*ObjectStore::retrieve(handle1), 50); + ASSERT_EQ(*ObjectStore::retrieve(handle2), 100); + store2.reset(); + ASSERT_EQ(*ObjectStore::retrieve(handle1), 50); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle2)); + store1.reset(); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle1)); + ASSERT_ANY_THROW(ObjectStore::retrieve(handle2)); +} diff --git a/cpp/core/utils/ObjectStore.cc b/cpp/core/utils/ObjectStore.cc index 5dc9c7843a73..648163e4bcfb 100644 --- a/cpp/core/utils/ObjectStore.cc +++ b/cpp/core/utils/ObjectStore.cc @@ -22,24 +22,22 @@ gluten::ObjectStore::~ObjectStore() { // destructing in reversed order (the last added object destructed first) const std::lock_guard lock(mtx_); - for (auto itr = aliveObjectHandles_.rbegin(); itr != aliveObjectHandles_.rend(); itr++) { + for (auto itr = aliveObjects_.rbegin(); itr != aliveObjects_.rend(); itr++) { ResourceHandle handle = *itr; - if (store_.lookup(handle) == nullptr) { - LOG(WARNING) << "Fatal: resource handle " + std::to_string(handle) + " not found in store."; - } store_.erase(handle); } + stores().erase(storeId_); } -gluten::ResourceHandle gluten::ObjectStore::save(std::shared_ptr obj) { +gluten::ObjectHandle gluten::ObjectStore::save(std::shared_ptr obj) { const std::lock_guard lock(mtx_); ResourceHandle handle = store_.insert(std::move(obj)); - aliveObjectHandles_.insert(handle); - return handle; + aliveObjects_.insert(handle); + return toObjHandle(handle); } -void gluten::ObjectStore::release(gluten::ResourceHandle handle) { +void gluten::ObjectStore::release0(gluten::ResourceHandle handle) { const std::lock_guard lock(mtx_); store_.erase(handle); - aliveObjectHandles_.erase(handle); + aliveObjects_.erase(handle); } diff --git a/cpp/core/utils/ObjectStore.h b/cpp/core/utils/ObjectStore.h index 6bfecf599b0d..8e5dd250a03b 100644 --- a/cpp/core/utils/ObjectStore.h +++ b/cpp/core/utils/ObjectStore.h @@ -19,25 +19,73 @@ #include #include "utils/ResourceMap.h" -#include "utils/exception.h" namespace gluten { +// ObjectHandle is a signed int64 consisting of: +// 1. 1 - 32 bits is a signed int32 as the object store's ID; +// 2. 1st bit is always zero to be compatible with jlong; +// 3. 33 - 64 bits is an unsigned int32 as the object's ID; +// +// When the object is tended to be retrieved with its ObjectHandle, +// the program first finds its resident object store, then looks up +// for the object in the store. +using StoreHandle = int32_t; +using ObjectHandle = int64_t; +constexpr static ObjectHandle kInvalidObjectHandle = -1; + // A store for caching shared-ptrs and enlarging lifecycles of the ptrs to match lifecycle of the store itself by // default, and also serving release calls to release a ptr in advance. This is typically used in JNI scenario to bind // a shared-ptr's lifecycle to a Java-side object or some kind of resource manager. class ObjectStore { public: static std::unique_ptr create() { - return std::unique_ptr(new ObjectStore()); + static std::mutex mtx; + std::lock_guard lock(mtx); + StoreHandle nextId = stores().nextId(); + auto store = std::unique_ptr(new ObjectStore(nextId)); + StoreHandle storeId = safeCast(stores().insert(store.get())); + GLUTEN_CHECK(storeId == nextId, "Store ID mismatched, this should not happen"); + return store; + } + + static void release(ObjectHandle handle) { + ResourceHandle storeId = safeCast(handle >> (sizeof(ResourceHandle) * 8)); + ResourceHandle resourceId = safeCast(handle & std::numeric_limits::max()); + auto store = stores().lookup(storeId); + store->release0(resourceId); + } + + template + static std::shared_ptr retrieve(ObjectHandle handle) { + ResourceHandle storeId = safeCast(handle >> (sizeof(ResourceHandle) * 8)); + ResourceHandle resourceId = safeCast(handle & std::numeric_limits::max()); + auto store = stores().lookup(storeId); + return store->retrieve0(resourceId); } virtual ~ObjectStore(); - ResourceHandle save(std::shared_ptr obj); + StoreHandle id() { + return storeId_; + } + + ObjectHandle save(std::shared_ptr obj); + + private: + static ResourceMap& stores() { + static ResourceMap stores; + return stores; + } + + ObjectHandle toObjHandle(ResourceHandle rh) { + ObjectHandle prefix = static_cast(storeId_) << (sizeof(ResourceHandle) * 8); + ObjectHandle objHandle = prefix + rh; + return objHandle; + } template - std::shared_ptr retrieve(ResourceHandle handle) { + std::shared_ptr retrieve0(ResourceHandle handle) { const std::lock_guard lock(mtx_); std::shared_ptr object = store_.lookup(handle); // Programming carefully. This will lead to ub if wrong typename T was passed in. @@ -45,12 +93,12 @@ class ObjectStore { return casted; } - void release(ResourceHandle handle); + void release0(ResourceHandle handle); - private: - ObjectStore(){}; + ObjectStore(StoreHandle storeId) : storeId_(storeId){}; + StoreHandle storeId_; ResourceMap> store_; - std::set aliveObjectHandles_; + std::set aliveObjects_; std::mutex mtx_; }; } // namespace gluten diff --git a/cpp/core/utils/ResourceMap.h b/cpp/core/utils/ResourceMap.h index c47fbd9ecb39..580ad4f6f94b 100644 --- a/cpp/core/utils/ResourceMap.h +++ b/cpp/core/utils/ResourceMap.h @@ -17,13 +17,25 @@ #pragma once +#include +#include #include #include +#include "utils/exception.h" namespace gluten { +using ResourceHandle = uint32_t; +static_assert(std::numeric_limits::min() == 0); -using ResourceHandle = int64_t; -constexpr static ResourceHandle kInvalidResourceHandle = -1; +template +T safeCast(F f) { + GLUTEN_CHECK(sizeof(T) <= sizeof(F), "Vain safe casting"); + F min = 0; + F max = static_cast(std::numeric_limits::max()); + GLUTEN_CHECK(f >= min, "Safe casting a negative number"); + GLUTEN_CHECK(f <= max, "Number overflow"); + return static_cast(f); +} /** * An utility class that map resource handle to its shared pointers. @@ -36,40 +48,48 @@ class ResourceMap { ResourceMap() : resourceId_(kInitResourceId) {} ResourceHandle insert(TResource holder) { - ResourceHandle result = resourceId_++; + ResourceHandle result = safeCast(resourceId_++); + const std::lock_guard lock(mtx_); map_.insert(std::pair(result, holder)); return result; } void erase(ResourceHandle moduleId) { - map_.erase(moduleId); + const std::lock_guard lock(mtx_); + GLUTEN_CHECK(map_.erase(moduleId) == 1, "Module not found in resource map: " + std::to_string(moduleId)); } TResource lookup(ResourceHandle moduleId) { + const std::lock_guard lock(mtx_); auto it = map_.find(moduleId); - if (it != map_.end()) { - return it->second; - } - return nullptr; + GLUTEN_CHECK(it != map_.end(), "Module not found in resource map: " + std::to_string(moduleId)); + return it->second; } void clear() { + const std::lock_guard lock(mtx_); map_.clear(); } size_t size() { + const std::lock_guard lock(mtx_); return map_.size(); } + size_t nextId() { + return resourceId_; + } + private: // Initialize the resource id starting value to a number greater than zero // to allow for easier debugging of uninitialized java variables. - static constexpr int kInitResourceId = 4; + static constexpr size_t kInitResourceId = 4; - ResourceHandle resourceId_; + std::atomic resourceId_{0}; // map from resource ids returned to Java and resource pointers std::unordered_map map_; + std::mutex mtx_; }; } // namespace gluten diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc index d8c8c0c24a94..87d77b7154f2 100644 --- a/cpp/velox/benchmarks/GenericBenchmark.cc +++ b/cpp/velox/benchmarks/GenericBenchmark.cc @@ -98,11 +98,8 @@ void setUpBenchmark(::benchmark::internal::Benchmark* bm) { } } -std::shared_ptr createShuffleWriter( - Runtime* runtime, - VeloxMemoryManager* memoryManager, - const std::string& dataFile, - const std::vector& localDirs) { +std::shared_ptr +createShuffleWriter(Runtime* runtime, const std::string& dataFile, const std::vector& localDirs) { PartitionWriterOptions partitionWriterOptions{}; // Configure compression. @@ -131,13 +128,13 @@ std::shared_ptr createShuffleWriter( partitionWriter = std::make_unique( FLAGS_shuffle_partitions, std::move(partitionWriterOptions), - memoryManager->getArrowMemoryPool(), + runtime->memoryManager()->getArrowMemoryPool(), std::move(rssClient)); } else { partitionWriter = std::make_unique( FLAGS_shuffle_partitions, std::move(partitionWriterOptions), - memoryManager->getArrowMemoryPool(), + runtime->memoryManager()->getArrowMemoryPool(), dataFile, localDirs); } @@ -147,8 +144,8 @@ std::shared_ptr createShuffleWriter( if (FLAGS_shuffle_writer == "sort") { options.shuffleWriterType = gluten::kSortShuffle; } - auto shuffleWriter = runtime->createShuffleWriter( - FLAGS_shuffle_partitions, std::move(partitionWriter), std::move(options), memoryManager); + auto shuffleWriter = + runtime->createShuffleWriter(FLAGS_shuffle_partitions, std::move(partitionWriter), std::move(options)); return std::reinterpret_pointer_cast(shuffleWriter); } @@ -178,7 +175,6 @@ void setCpu(::benchmark::State& state) { void runShuffle( Runtime* runtime, - VeloxMemoryManager* memoryManager, BenchmarkAllocationListener* listener, const std::shared_ptr& resultIter, WriterMetrics& metrics) { @@ -187,7 +183,7 @@ void runShuffle( bool isFromEnv; GLUTEN_THROW_NOT_OK(setLocalDirsAndDataFileFromEnv(dataFile, localDirs, isFromEnv)); - auto shuffleWriter = createShuffleWriter(runtime, memoryManager, dataFile, localDirs); + auto shuffleWriter = createShuffleWriter(runtime, dataFile, localDirs); listener->setShuffleWriter(shuffleWriter.get()); int64_t totalTime = 0; @@ -226,18 +222,19 @@ void updateBenchmarkMetrics( } // namespace +using RuntimeFactory = std::function listener)>; + auto BM_Generic = [](::benchmark::State& state, const std::string& planFile, const std::vector& splitFiles, const std::vector& dataFiles, - Runtime* runtime, + RuntimeFactory runtimeFactory, FileReaderType readerType) { setCpu(state); auto listener = std::make_unique(FLAGS_memory_limit); auto* listenerPtr = listener.get(); - auto memoryManager = std::make_unique( - "generic_benchmark", gluten::defaultMemoryAllocator(), std::move(listener)); + auto runtime = runtimeFactory(std::move(listener)); auto plan = getPlanFromFile("Plan", planFile); std::vector splits{}; @@ -271,18 +268,17 @@ auto BM_Generic = [](::benchmark::State& state, for (auto& split : splits) { runtime->parseSplitInfo(reinterpret_cast(split.data()), split.size(), std::nullopt); } - auto resultIter = runtime->createResultIterator( - memoryManager.get(), "/tmp/test-spill", std::move(inputIters), runtime->getConfMap()); + auto resultIter = runtime->createResultIterator("/tmp/test-spill", std::move(inputIters), runtime->getConfMap()); listenerPtr->setIterator(resultIter.get()); if (FLAGS_with_shuffle) { - runShuffle(runtime, memoryManager.get(), listenerPtr, resultIter, writerMetrics); + runShuffle(runtime, listenerPtr, resultIter, writerMetrics); } else { // May write the output into file. auto veloxPlan = dynamic_cast(runtime)->getVeloxPlan(); ArrowSchema cSchema; - toArrowSchema(veloxPlan->outputType(), memoryManager->getLeafMemoryPool().get(), &cSchema); + toArrowSchema(veloxPlan->outputType(), runtime->memoryManager()->getLeafMemoryPool().get(), &cSchema); GLUTEN_ASSIGN_OR_THROW(auto outputSchema, arrow::ImportSchema(&cSchema)); ArrowWriter writer{FLAGS_save_output}; state.PauseTiming(); @@ -328,33 +324,36 @@ auto BM_Generic = [](::benchmark::State& state, } updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics); + Runtime::release(runtime); }; -auto BM_ShuffleWrite = - [](::benchmark::State& state, const std::string& inputFile, Runtime* runtime, FileReaderType readerType) { - setCpu(state); - - auto listener = std::make_unique(FLAGS_memory_limit); - auto* listenerPtr = listener.get(); - auto memoryManager = std::make_unique( - "generic_benchmark", gluten::defaultMemoryAllocator(), std::move(listener)); - - WriterMetrics writerMetrics{}; - int64_t readInputTime = 0; - int64_t elapsedTime = 0; - { - ScopedTimer timer(&elapsedTime); - for (auto _ : state) { - auto resultIter = getInputIteratorFromFileReader(inputFile, readerType); - runShuffle(runtime, memoryManager.get(), listenerPtr, resultIter, writerMetrics); - - auto reader = static_cast(resultIter->getInputIter()); - readInputTime += reader->getCollectBatchTime(); - } - } +auto BM_ShuffleWrite = [](::benchmark::State& state, + const std::string& inputFile, + RuntimeFactory runtimeFactory, + FileReaderType readerType) { + setCpu(state); - updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics); - }; + auto listener = std::make_unique(FLAGS_memory_limit); + auto* listenerPtr = listener.get(); + auto runtime = runtimeFactory(std::move(listener)); + + WriterMetrics writerMetrics{}; + int64_t readInputTime = 0; + int64_t elapsedTime = 0; + { + ScopedTimer timer(&elapsedTime); + for (auto _ : state) { + auto resultIter = getInputIteratorFromFileReader(inputFile, readerType); + runShuffle(runtime, listenerPtr, resultIter, writerMetrics); + + auto reader = static_cast(resultIter->getInputIter()); + readInputTime += reader->getCollectBatchTime(); + } + } + + updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics); + Runtime::release(runtime); +}; int main(int argc, char** argv) { ::benchmark::Initialize(&argc, argv); @@ -512,22 +511,26 @@ int main(int argc, char** argv) { } } - auto runtime = Runtime::create(kVeloxRuntimeKind, sessionConf); + RuntimeFactory runtimeFactory = [=](std::unique_ptr listener) { + return dynamic_cast(Runtime::create(kVeloxRuntimeKind, std::move(listener), sessionConf)); + }; #define GENERIC_BENCHMARK(READER_TYPE) \ do { \ - auto* bm = ::benchmark::RegisterBenchmark( \ - "GenericBenchmark", BM_Generic, substraitJsonFile, splitFiles, dataFiles, runtime, READER_TYPE) \ - ->MeasureProcessCPUTime() \ - ->UseRealTime(); \ + auto* bm = \ + ::benchmark::RegisterBenchmark( \ + "GenericBenchmark", BM_Generic, substraitJsonFile, splitFiles, dataFiles, runtimeFactory, READER_TYPE) \ + ->MeasureProcessCPUTime() \ + ->UseRealTime(); \ setUpBenchmark(bm); \ } while (0) #define SHUFFLE_WRITE_BENCHMARK(READER_TYPE) \ do { \ - auto* bm = ::benchmark::RegisterBenchmark("ShuffleWrite", BM_ShuffleWrite, dataFiles[0], runtime, READER_TYPE) \ - ->MeasureProcessCPUTime() \ - ->UseRealTime(); \ + auto* bm = \ + ::benchmark::RegisterBenchmark("ShuffleWrite", BM_ShuffleWrite, dataFiles[0], runtimeFactory, READER_TYPE) \ + ->MeasureProcessCPUTime() \ + ->UseRealTime(); \ setUpBenchmark(bm); \ } while (0) @@ -561,7 +564,6 @@ int main(int argc, char** argv) { ::benchmark::RunSpecifiedBenchmarks(); ::benchmark::Shutdown(); - Runtime::release(runtime); gluten::VeloxBackend::get()->tearDown(); return 0; diff --git a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc index 894c35351f17..45348ed4a63b 100644 --- a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc +++ b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc @@ -257,7 +257,7 @@ class GoogleBenchmarkVeloxParquetWriteCacheScanBenchmark : public GoogleBenchmar // reuse the ParquetWriteConverter for batches caused system % increase a lot auto fileName = "velox_parquet_write.parquet"; - auto runtime = Runtime::create(kVeloxRuntimeKind); + auto runtime = Runtime::create(kVeloxRuntimeKind, AllocationListener::noop()); auto memoryManager = getDefaultMemoryManager(); auto veloxPool = memoryManager->getAggregateMemoryPool(); diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 1ec5879966d6..a3658faa3a18 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -55,8 +55,10 @@ using namespace facebook; namespace gluten { namespace { -gluten::Runtime* veloxRuntimeFactory(const std::unordered_map& sessionConf) { - return new gluten::VeloxRuntime(sessionConf); +gluten::Runtime* veloxRuntimeFactory( + std::unique_ptr listener, + const std::unordered_map& sessionConf) { + return new gluten::VeloxRuntime(std::move(listener), sessionConf); } } // namespace diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index 738ce99a3bc7..fde87b5b49bd 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -56,8 +56,12 @@ using namespace facebook; namespace gluten { -VeloxRuntime::VeloxRuntime(const std::unordered_map& confMap) : Runtime(confMap) { +VeloxRuntime::VeloxRuntime( + std::unique_ptr listener, + const std::unordered_map& confMap) + : Runtime(std::make_shared(std::move(listener)), confMap) { // Refresh session config. + vmm_ = dynamic_cast(memoryManager_.get()); veloxCfg_ = std::make_shared(confMap_); debugModeEnabled_ = veloxCfg_->get(kDebugModeEnabled, false); FLAGS_minloglevel = veloxCfg_->get(kGlogSeverityLevel, FLAGS_minloglevel); @@ -127,15 +131,17 @@ void VeloxRuntime::injectWriteFilesTempPath(const std::string& path) { writeFilesTempPath_ = path; } +VeloxMemoryManager* VeloxRuntime::memoryManager() { + return vmm_; +} + std::shared_ptr VeloxRuntime::createResultIterator( - MemoryManager* memoryManager, const std::string& spillDir, const std::vector>& inputs, const std::unordered_map& sessionConf) { LOG_IF(INFO, debugModeEnabled_) << "VeloxRuntime session config:" << printConfig(confMap_); - VeloxPlanConverter veloxPlanConverter( - inputs, getLeafVeloxPool(memoryManager).get(), sessionConf, writeFilesTempPath_); + VeloxPlanConverter veloxPlanConverter(inputs, vmm_->getLeafMemoryPool().get(), sessionConf, writeFilesTempPath_); veloxPlan_ = veloxPlanConverter.toVeloxPlan(substraitPlan_, std::move(localFiles_)); // Scan node can be required. @@ -146,15 +152,14 @@ std::shared_ptr VeloxRuntime::createResultIterator( // Separate the scan ids and stream ids, and get the scan infos. getInfoAndIds(veloxPlanConverter.splitInfos(), veloxPlan_->leafPlanNodeIds(), scanInfos, scanIds, streamIds); - auto* vmm = toVeloxMemoryManager(memoryManager); auto wholestageIter = std::make_unique( - vmm, veloxPlan_, scanIds, scanInfos, streamIds, spillDir, sessionConf, taskInfo_); + vmm_, veloxPlan_, scanIds, scanInfos, streamIds, spillDir, sessionConf, taskInfo_); return std::make_shared(std::move(wholestageIter), this); } -std::shared_ptr VeloxRuntime::createColumnar2RowConverter(MemoryManager* memoryManager) { - auto ctxVeloxPool = getLeafVeloxPool(memoryManager); - return std::make_shared(ctxVeloxPool); +std::shared_ptr VeloxRuntime::createColumnar2RowConverter() { + auto veloxPool = vmm_->getLeafMemoryPool(); + return std::make_shared(veloxPool); } std::shared_ptr VeloxRuntime::createOrGetEmptySchemaBatch(int32_t numRows) { @@ -167,51 +172,45 @@ std::shared_ptr VeloxRuntime::createOrGetEmptySchemaBatch(int32_t } std::shared_ptr VeloxRuntime::select( - MemoryManager* memoryManager, std::shared_ptr batch, std::vector columnIndices) { - auto ctxVeloxPool = getLeafVeloxPool(memoryManager); - auto veloxBatch = gluten::VeloxColumnarBatch::from(ctxVeloxPool.get(), batch); - auto outputBatch = veloxBatch->select(ctxVeloxPool.get(), std::move(columnIndices)); + auto veloxPool = vmm_->getLeafMemoryPool(); + auto veloxBatch = gluten::VeloxColumnarBatch::from(veloxPool.get(), batch); + auto outputBatch = veloxBatch->select(veloxPool.get(), std::move(columnIndices)); return outputBatch; } -std::shared_ptr VeloxRuntime::createRow2ColumnarConverter( - MemoryManager* memoryManager, - struct ArrowSchema* cSchema) { - auto ctxVeloxPool = getLeafVeloxPool(memoryManager); - return std::make_shared(cSchema, ctxVeloxPool); +std::shared_ptr VeloxRuntime::createRow2ColumnarConverter(struct ArrowSchema* cSchema) { + auto veloxPool = vmm_->getLeafMemoryPool(); + return std::make_shared(cSchema, veloxPool); } std::shared_ptr VeloxRuntime::createShuffleWriter( int numPartitions, std::unique_ptr partitionWriter, - ShuffleWriterOptions options, - MemoryManager* memoryManager) { - auto ctxPool = getLeafVeloxPool(memoryManager); - auto arrowPool = memoryManager->getArrowMemoryPool(); - std::shared_ptr shuffleWriter; + ShuffleWriterOptions options) { + auto veloxPool = vmm_->getLeafMemoryPool(); + auto arrowPool = vmm_->getArrowMemoryPool(); GLUTEN_ASSIGN_OR_THROW( - shuffleWriter, + std::shared_ptr shuffleWriter, VeloxShuffleWriter::create( options.shuffleWriterType, numPartitions, std::move(partitionWriter), std::move(options), - ctxPool, + veloxPool, arrowPool)); return shuffleWriter; } std::shared_ptr VeloxRuntime::createDatasource( const std::string& filePath, - MemoryManager* memoryManager, std::shared_ptr schema) { static std::atomic_uint32_t id{0UL}; - auto veloxPool = getAggregateVeloxPool(memoryManager)->addAggregateChild("datasource." + std::to_string(id++)); + auto veloxPool = vmm_->getAggregateMemoryPool()->addAggregateChild("datasource." + std::to_string(id++)); // Pass a dedicate pool for S3 and GCS sinks as can't share veloxPool // with parquet writer. - auto sinkPool = getLeafVeloxPool(memoryManager); + auto sinkPool = vmm_->getLeafMemoryPool(); if (isSupportedHDFSPath(filePath)) { #ifdef ENABLE_HDFS return std::make_shared(filePath, veloxPool, sinkPool, schema); @@ -246,12 +245,10 @@ std::shared_ptr VeloxRuntime::createDatasource( std::shared_ptr VeloxRuntime::createShuffleReader( std::shared_ptr schema, - ShuffleReaderOptions options, - arrow::MemoryPool* pool, - MemoryManager* memoryManager) { + ShuffleReaderOptions options) { auto rowType = facebook::velox::asRowType(gluten::fromArrowSchema(schema)); auto codec = gluten::createArrowIpcCodec(options.compressionType, options.codecBackend); - auto ctxVeloxPool = getLeafVeloxPool(memoryManager); + auto ctxVeloxPool = vmm_->getLeafMemoryPool(); auto veloxCompressionType = facebook::velox::common::stringToCompressionKind(options.compressionTypeStr); auto deserializerFactory = std::make_unique( schema, @@ -259,19 +256,17 @@ std::shared_ptr VeloxRuntime::createShuffleReader( veloxCompressionType, rowType, options.batchSize, - pool, + vmm_->getArrowMemoryPool(), ctxVeloxPool, options.shuffleWriterType); auto reader = std::make_shared(std::move(deserializerFactory)); return reader; } -std::unique_ptr VeloxRuntime::createColumnarBatchSerializer( - MemoryManager* memoryManager, - arrow::MemoryPool* arrowPool, - struct ArrowSchema* cSchema) { - auto ctxVeloxPool = getLeafVeloxPool(memoryManager); - return std::make_unique(arrowPool, ctxVeloxPool, cSchema); +std::unique_ptr VeloxRuntime::createColumnarBatchSerializer(struct ArrowSchema* cSchema) { + auto arrowPool = vmm_->getArrowMemoryPool(); + auto veloxPool = vmm_->getLeafMemoryPool(); + return std::make_unique(arrowPool, veloxPool, cSchema); } void VeloxRuntime::dumpConf(const std::string& path) { diff --git a/cpp/velox/compute/VeloxRuntime.h b/cpp/velox/compute/VeloxRuntime.h index 80408bccb2b5..096ecb6fbf13 100644 --- a/cpp/velox/compute/VeloxRuntime.h +++ b/cpp/velox/compute/VeloxRuntime.h @@ -33,81 +33,49 @@ inline static const std::string kVeloxRuntimeKind{"velox"}; class VeloxRuntime final : public Runtime { public: - explicit VeloxRuntime(const std::unordered_map& confMap); + explicit VeloxRuntime( + std::unique_ptr listener, + const std::unordered_map& confMap); void parsePlan(const uint8_t* data, int32_t size, std::optional dumpFile) override; void parseSplitInfo(const uint8_t* data, int32_t size, std::optional dumpFile) override; - static std::shared_ptr getAggregateVeloxPool(MemoryManager* memoryManager) { - return toVeloxMemoryManager(memoryManager)->getAggregateMemoryPool(); - } - - static std::shared_ptr getLeafVeloxPool(MemoryManager* memoryManager) { - return toVeloxMemoryManager(memoryManager)->getLeafMemoryPool(); - } - - static VeloxMemoryManager* toVeloxMemoryManager(MemoryManager* memoryManager) { - if (auto veloxMemoryManager = dynamic_cast(memoryManager)) { - return veloxMemoryManager; - } else { - GLUTEN_CHECK(false, "Velox memory manager should be used for Velox runtime."); - } - } - - MemoryManager* createMemoryManager( - const std::string& name, - std::shared_ptr allocator, - std::unique_ptr listener) override { - return new VeloxMemoryManager(name, allocator, std::move(listener)); - } + VeloxMemoryManager* memoryManager() override; // FIXME This is not thread-safe? std::shared_ptr createResultIterator( - MemoryManager* memoryManager, const std::string& spillDir, const std::vector>& inputs = {}, const std::unordered_map& sessionConf = {}) override; - std::shared_ptr createColumnar2RowConverter(MemoryManager* memoryManager) override; + std::shared_ptr createColumnar2RowConverter() override; std::shared_ptr createOrGetEmptySchemaBatch(int32_t numRows) override; - std::shared_ptr select( - MemoryManager* memoryManager, - std::shared_ptr batch, - std::vector columnIndices) override; + std::shared_ptr select(std::shared_ptr batch, std::vector columnIndices) + override; - std::shared_ptr createRow2ColumnarConverter( - MemoryManager* memoryManager, - struct ArrowSchema* cSchema) override; + std::shared_ptr createRow2ColumnarConverter(struct ArrowSchema* cSchema) override; std::shared_ptr createShuffleWriter( int numPartitions, std::unique_ptr partitionWriter, - ShuffleWriterOptions options, - MemoryManager* memoryManager) override; + ShuffleWriterOptions options) override; Metrics* getMetrics(ColumnarBatchIterator* rawIter, int64_t exportNanos) override { auto iter = static_cast(rawIter); return iter->getMetrics(exportNanos); } - std::shared_ptr createDatasource( - const std::string& filePath, - MemoryManager* memoryManager, - std::shared_ptr schema) override; + std::shared_ptr createDatasource(const std::string& filePath, std::shared_ptr schema) + override; std::shared_ptr createShuffleReader( std::shared_ptr schema, - ShuffleReaderOptions options, - arrow::MemoryPool* pool, - MemoryManager* memoryManager) override; + ShuffleReaderOptions options) override; - std::unique_ptr createColumnarBatchSerializer( - MemoryManager* memoryManager, - arrow::MemoryPool* arrowPool, - struct ArrowSchema* cSchema) override; + std::unique_ptr createColumnarBatchSerializer(struct ArrowSchema* cSchema) override; std::string planString(bool details, const std::unordered_map& sessionConf) override; @@ -131,6 +99,7 @@ class VeloxRuntime final : public Runtime { std::vector& streamIds); private: + VeloxMemoryManager* vmm_; std::shared_ptr veloxPlan_; std::shared_ptr veloxCfg_; bool debugModeEnabled_{false}; diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 5920a08985f4..cbc6c838b1b7 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -83,8 +83,13 @@ WholeStageResultIterator::WholeStageResultIterator( std::unordered_set emptySet; velox::core::PlanFragment planFragment{planNode, velox::core::ExecutionStrategy::kUngrouped, 1, emptySet}; std::shared_ptr queryCtx = createNewVeloxQueryCtx(); + static std::atomic vtId{0}; // Velox task ID to distinguish from Spark task ID. task_ = velox::exec::Task::create( - fmt::format("Gluten_Stage_{}_TID_{}", std::to_string(taskInfo_.stageId), std::to_string(taskInfo_.taskId)), + fmt::format( + "Gluten_Stage_{}_TID_{}_VTID_{}", + std::to_string(taskInfo_.stageId), + std::to_string(taskInfo_.taskId), + std::to_string(vtId++)), std::move(planFragment), 0, std::move(queryCtx), diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index 3b52eaa86b2f..f11edf1a435d 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -30,6 +30,7 @@ #include "jni/JniFileSystem.h" #include "memory/VeloxMemoryManager.h" #include "substrait/SubstraitToVeloxPlanValidator.h" +#include "utils/ObjectStore.h" #include "utils/VeloxBatchAppender.h" #include "velox/common/base/BloomFilter.h" @@ -159,8 +160,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_ auto filter = std::make_shared>>(); filter->reset(capacity); GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized"); - return ctx->objectStore()->save(filter); - JNI_METHOD_END(gluten::kInvalidResourceHandle) + return ctx->saveObject(filter); + JNI_METHOD_END(gluten::kInvalidObjectHandle) } JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_init( // NOLINT @@ -173,8 +174,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_ auto filter = std::make_shared>>(); uint8_t* serialized = safeArray.elems(); filter->merge(reinterpret_cast(serialized)); - return ctx->objectStore()->save(filter); - JNI_METHOD_END(gluten::kInvalidResourceHandle) + return ctx->saveObject(filter); + JNI_METHOD_END(gluten::kInvalidObjectHandle) } JNIEXPORT void JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_insertLong( // NOLINT @@ -184,7 +185,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_i jlong item) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto filter = ctx->objectStore()->retrieve>>(handle); + auto filter = gluten::ObjectStore::retrieve>>(handle); GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized"); filter->insert(folly::hasher()(item)); JNI_METHOD_END() @@ -197,7 +198,7 @@ JNIEXPORT jboolean JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapp jlong item) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto filter = ctx->objectStore()->retrieve>>(handle); + auto filter = gluten::ObjectStore::retrieve>>(handle); GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized"); bool out = filter->mayContain(folly::hasher()(item)); return out; @@ -222,8 +223,8 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_m jlong other) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto to = ctx->objectStore()->retrieve>>(handle); - auto from = ctx->objectStore()->retrieve>>(other); + auto to = gluten::ObjectStore::retrieve>>(handle); + auto from = gluten::ObjectStore::retrieve>>(other); GLUTEN_CHECK(to->isSet(), "Bloom-filter is not initialized"); GLUTEN_CHECK(from->isSet(), "Bloom-filter is not initialized"); std::vector serialized = serialize(from.get()); @@ -237,7 +238,7 @@ JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWra jlong handle) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto filter = ctx->objectStore()->retrieve>>(handle); + auto filter = gluten::ObjectStore::retrieve>>(handle); GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized"); std::vector buffer = serialize(filter.get()); auto size = buffer.capacity(); @@ -250,18 +251,16 @@ JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWra JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBatchAppenderJniWrapper_create( // NOLINT JNIEnv* env, jobject wrapper, - jlong memoryManagerHandle, jint minOutputBatchSize, jobject jIter) { JNI_METHOD_START auto ctx = gluten::getRuntime(env, wrapper); - auto memoryManager = jniCastOrThrow(memoryManagerHandle); - auto pool = gluten::VeloxRuntime::getLeafVeloxPool(memoryManager); + auto pool = dynamic_cast(ctx->memoryManager())->getLeafMemoryPool(); auto iter = gluten::makeJniColumnarBatchIterator(env, jIter, ctx, nullptr); auto appender = std::make_shared( std::make_unique(pool.get(), minOutputBatchSize, std::move(iter))); - return ctx->objectStore()->save(appender); - JNI_METHOD_END(gluten::kInvalidResourceHandle) + return ctx->saveObject(appender); + JNI_METHOD_END(gluten::kInvalidObjectHandle) } #ifdef __cplusplus diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index efd165b736be..442090004a41 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -161,17 +161,14 @@ class ArbitratorFactoryRegister { gluten::AllocationListener* listener_; }; -VeloxMemoryManager::VeloxMemoryManager( - const std::string& name, - std::shared_ptr allocator, - std::unique_ptr listener) - : MemoryManager(), name_(name), listener_(std::move(listener)) { +VeloxMemoryManager::VeloxMemoryManager(std::unique_ptr listener) + : MemoryManager(), listener_(std::move(listener)) { auto reservationBlockSize = VeloxBackend::get()->getBackendConf()->get( kMemoryReservationBlockSize, kMemoryReservationBlockSizeDefault); auto memInitCapacity = VeloxBackend::get()->getBackendConf()->get(kVeloxMemInitCapacity, kVeloxMemInitCapacityDefault); blockListener_ = std::make_unique(listener_.get(), reservationBlockSize); - listenableAlloc_ = std::make_unique(allocator.get(), blockListener_.get()); + listenableAlloc_ = std::make_unique(defaultMemoryAllocator().get(), blockListener_.get()); arrowPool_ = std::make_unique(listenableAlloc_.get()); ArbitratorFactoryRegister afr(listener_.get()); @@ -189,11 +186,11 @@ VeloxMemoryManager::VeloxMemoryManager( veloxMemoryManager_ = std::make_unique(mmOptions); veloxAggregatePool_ = veloxMemoryManager_->addRootPool( - name_ + "_root", + "root", velox::memory::kMaxMemory, // the 3rd capacity facebook::velox::memory::MemoryReclaimer::create()); - veloxLeafPool_ = veloxAggregatePool_->addLeafChild(name_ + "_default_leaf"); + veloxLeafPool_ = veloxAggregatePool_->addLeafChild("default_leaf"); } namespace { @@ -317,8 +314,10 @@ bool VeloxMemoryManager::tryDestructSafe() { VeloxMemoryManager::~VeloxMemoryManager() { static const uint32_t kWaitTimeoutMs = FLAGS_gluten_velox_aysnc_timeout_on_task_stopping; // 30s by default uint32_t accumulatedWaitMs = 0UL; + bool destructed = false; for (int32_t tryCount = 0; accumulatedWaitMs < kWaitTimeoutMs; tryCount++) { - if (tryDestructSafe()) { + destructed = tryDestructSafe(); + if (destructed) { if (tryCount > 0) { LOG(INFO) << "All the outstanding memory resources successfully released. "; } @@ -330,6 +329,10 @@ VeloxMemoryManager::~VeloxMemoryManager() { usleep(waitMs * 1000); accumulatedWaitMs += waitMs; } + if (!destructed) { + LOG(ERROR) << "Failed to release Velox memory manager after " << accumulatedWaitMs + << "ms as there are still outstanding memory resources. "; + } #ifdef ENABLE_JEMALLOC je_gluten_malloc_stats_print(NULL, NULL, NULL); #endif diff --git a/cpp/velox/memory/VeloxMemoryManager.h b/cpp/velox/memory/VeloxMemoryManager.h index 3ba5bbf7d25f..3607ca793f3e 100644 --- a/cpp/velox/memory/VeloxMemoryManager.h +++ b/cpp/velox/memory/VeloxMemoryManager.h @@ -27,10 +27,7 @@ namespace gluten { class VeloxMemoryManager final : public MemoryManager { public: - VeloxMemoryManager( - const std::string& name, - std::shared_ptr allocator, - std::unique_ptr listener); + VeloxMemoryManager(std::unique_ptr listener); ~VeloxMemoryManager() override; VeloxMemoryManager(const VeloxMemoryManager&) = delete; @@ -72,8 +69,6 @@ class VeloxMemoryManager final : public MemoryManager { private: bool tryDestructSafe(); - std::string name_; - #ifdef GLUTEN_ENABLE_HBM std::unique_ptr wrappedAlloc_; #endif @@ -92,8 +87,7 @@ class VeloxMemoryManager final : public MemoryManager { /// Not tracked by Spark and should only used in test or validation. inline std::shared_ptr getDefaultMemoryManager() { - static auto memoryManager = std::make_shared( - "test", gluten::defaultMemoryAllocator(), gluten::AllocationListener::noop()); + static auto memoryManager = std::make_shared(gluten::AllocationListener::noop()); return memoryManager; } diff --git a/cpp/velox/tests/MemoryManagerTest.cc b/cpp/velox/tests/MemoryManagerTest.cc index 400beafcc1f8..52f2fa8b661c 100644 --- a/cpp/velox/tests/MemoryManagerTest.cc +++ b/cpp/velox/tests/MemoryManagerTest.cc @@ -54,7 +54,7 @@ class MemoryManagerTest : public ::testing::Test { } void SetUp() override { - vmm_ = std::make_unique("test", stdAllocator_, std::make_unique()); + vmm_ = std::make_unique(std::make_unique()); listener_ = vmm_->getListener(); allocator_ = vmm_->allocator(); } diff --git a/cpp/velox/tests/RuntimeTest.cc b/cpp/velox/tests/RuntimeTest.cc index 377d76054265..563539d7d63e 100644 --- a/cpp/velox/tests/RuntimeTest.cc +++ b/cpp/velox/tests/RuntimeTest.cc @@ -18,19 +18,36 @@ #include "compute/VeloxRuntime.h" #include +#include "compute/VeloxBackend.h" namespace gluten { +class DummyMemoryManager final : public MemoryManager { + public: + arrow::MemoryPool* getArrowMemoryPool() override { + throw GlutenException("Not yet implemented"); + } + const MemoryUsageStats collectMemoryUsageStats() const override { + throw GlutenException("Not yet implemented"); + } + const int64_t shrink(int64_t size) override { + throw GlutenException("Not yet implemented"); + } + void hold() override { + throw GlutenException("Not yet implemented"); + } +}; + class DummyRuntime final : public Runtime { public: - DummyRuntime(const std::unordered_map& conf) : Runtime(conf) {} + DummyRuntime(std::unique_ptr listener, const std::unordered_map& conf) + : Runtime(std::make_shared(), conf) {} void parsePlan(const uint8_t* data, int32_t size, std::optional dumpFile) override {} void parseSplitInfo(const uint8_t* data, int32_t size, std::optional dumpFile) override {} std::shared_ptr createResultIterator( - MemoryManager* memoryManager, const std::string& spillDir, const std::vector>& inputs, const std::unordered_map& sessionConf) override { @@ -38,54 +55,41 @@ class DummyRuntime final : public Runtime { auto iter = std::make_shared(std::move(resIter)); return iter; } - MemoryManager* createMemoryManager( - const std::string& name, - std::shared_ptr ptr, - std::unique_ptr uniquePtr) override { + MemoryManager* memoryManager() override { throw GlutenException("Not yet implemented"); } std::shared_ptr createOrGetEmptySchemaBatch(int32_t numRows) override { throw GlutenException("Not yet implemented"); } - std::shared_ptr createColumnar2RowConverter(MemoryManager* memoryManager) override { + std::shared_ptr createColumnar2RowConverter() override { throw GlutenException("Not yet implemented"); } - std::shared_ptr createRow2ColumnarConverter( - MemoryManager* memoryManager, - struct ArrowSchema* cSchema) override { + std::shared_ptr createRow2ColumnarConverter(struct ArrowSchema* cSchema) override { throw GlutenException("Not yet implemented"); } std::shared_ptr createShuffleWriter( int numPartitions, std::unique_ptr partitionWriter, - ShuffleWriterOptions, - MemoryManager* memoryManager) override { + ShuffleWriterOptions) override { throw GlutenException("Not yet implemented"); } Metrics* getMetrics(ColumnarBatchIterator* rawIter, int64_t exportNanos) override { static Metrics m(1); return &m; } - std::shared_ptr createDatasource( - const std::string& filePath, - MemoryManager* memoryManager, - std::shared_ptr schema) override { + std::shared_ptr createDatasource(const std::string& filePath, std::shared_ptr schema) + override { throw GlutenException("Not yet implemented"); } std::shared_ptr createShuffleReader( std::shared_ptr schema, - ShuffleReaderOptions options, - arrow::MemoryPool* pool, - MemoryManager* memoryManager) override { + ShuffleReaderOptions options) override { throw GlutenException("Not yet implemented"); } - std::unique_ptr createColumnarBatchSerializer( - MemoryManager* memoryManager, - arrow::MemoryPool* arrowPool, - struct ArrowSchema* cSchema) override { + std::unique_ptr createColumnarBatchSerializer(struct ArrowSchema* cSchema) override { throw GlutenException("Not yet implemented"); } - std::shared_ptr select(MemoryManager*, std::shared_ptr, std::vector) override { + std::shared_ptr select(std::shared_ptr, std::vector) override { throw GlutenException("Not yet implemented"); } std::string planString(bool details, const std::unordered_map& sessionConf) override { @@ -100,8 +104,6 @@ class DummyRuntime final : public Runtime { } private: - ResourceMap> resultIteratorHolder_; - class DummyResultIterator : public ColumnarBatchIterator { public: std::shared_ptr next() override { @@ -118,20 +120,30 @@ class DummyRuntime final : public Runtime { }; }; -static Runtime* dummyRuntimeFactory(const std::unordered_map conf) { - return new DummyRuntime(conf); +static Runtime* dummyRuntimeFactory( + std::unique_ptr listener, + const std::unordered_map conf) { + return new DummyRuntime(std::move(listener), conf); } TEST(TestRuntime, CreateRuntime) { Runtime::registerFactory("DUMMY", dummyRuntimeFactory); - auto runtime = Runtime::create("DUMMY"); + auto runtime = Runtime::create("DUMMY", AllocationListener::noop()); ASSERT_EQ(typeid(*runtime), typeid(DummyRuntime)); Runtime::release(runtime); } +TEST(TestRuntime, CreateVeloxRuntime) { + VeloxBackend::create({}); + auto runtime = Runtime::create(kVeloxRuntimeKind, AllocationListener::noop()); + ASSERT_EQ(typeid(*runtime), typeid(VeloxRuntime)); + Runtime::release(runtime); +} + TEST(TestRuntime, GetResultIterator) { - auto runtime = std::make_shared(std::unordered_map()); - auto iter = runtime->createResultIterator(nullptr, "/tmp/test-spill", {}, {}); + auto runtime = + std::make_shared(AllocationListener::noop(), std::unordered_map()); + auto iter = runtime->createResultIterator("/tmp/test-spill", {}, {}); ASSERT_TRUE(iter->hasNext()); auto next = iter->next(); ASSERT_NE(next, nullptr); diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 97b3db5549bc..b812b6b52bd6 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -302,4 +302,4 @@ check_commit compile echo "Successfully built Velox from Source." -echo $TARGET_BUILD_SUMMARY >"${VELOX_HOME}/velox-build.cache" +echo $TARGET_BUILD_SUMMARY > "${VELOX_HOME}/velox-build.cache" diff --git a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala index 524a3ee2e464..a7836e4a13d1 100644 --- a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala +++ b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala @@ -19,9 +19,7 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings import org.apache.gluten.memory.alloc.CHNativeMemoryAllocators -import org.apache.gluten.memory.memtarget.MemoryTarget -import org.apache.gluten.memory.memtarget.Spiller -import org.apache.gluten.memory.memtarget.Spillers +import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.vectorized._ import org.apache.spark._ @@ -33,7 +31,6 @@ import org.apache.celeborn.client.ShuffleClient import org.apache.celeborn.common.CelebornConf import java.io.IOException -import java.util import java.util.Locale class CHCelebornHashBasedColumnarShuffleWriter[K, V]( @@ -85,7 +82,10 @@ class CHCelebornHashBasedColumnarShuffleWriter[K, V]( CHNativeMemoryAllocators.createSpillable( "CelebornShuffleWriter", new Spiller() { - override def spill(self: MemoryTarget, size: Long): Long = { + override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = { + if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { + return 0L + } if (nativeShuffleWriter == -1L) { throw new IllegalStateException( "Fatal: spill() called before a celeborn shuffle writer " + @@ -98,8 +98,6 @@ class CHCelebornHashBasedColumnarShuffleWriter[K, V]( logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data") spilled } - - override def applicablePhases(): util.Set[Spiller.Phase] = Spillers.PHASE_SET_SPILL_ONLY } ) } diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index 699626db12c5..1f125a164c8b 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -19,7 +19,6 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.vectorized._ @@ -65,7 +64,7 @@ private class CelebornColumnarBatchSerializerInstance( extends SerializerInstance with Logging { - private val nmm = NativeMemoryManagers.contextInstance("ShuffleReader") + private val runtime = Runtimes.contextInstance("CelebornShuffleReader") private val shuffleReaderHandle = { val allocator: BufferAllocator = ArrowBufferAllocators @@ -86,12 +85,11 @@ private class CelebornColumnarBatchSerializerInstance( GlutenConfig.getConf.columnarShuffleCodecBackend.orNull val shuffleWriterType = conf.get("spark.celeborn.client.spark.shuffle.writer", "hash").toLowerCase(Locale.ROOT) - val jniWrapper = ShuffleReaderJniWrapper.create() + val jniWrapper = ShuffleReaderJniWrapper.create(runtime) val batchSize = GlutenConfig.getConf.maxBatchSize val handle = jniWrapper .make( cSchema.memoryAddress(), - nmm.getNativeInstanceHandle, compressionCodec, compressionCodecBackend, batchSize, @@ -119,11 +117,10 @@ private class CelebornColumnarBatchSerializerInstance( with TaskResource { private val byteIn: JniByteInputStream = JniByteInputStreams.create(in) private val wrappedOut: GeneralOutIterator = new ColumnarBatchOutIterator( - Runtimes.contextInstance(), + runtime, ShuffleReaderJniWrapper - .create() - .readStream(shuffleReaderHandle, byteIn), - nmm) + .create(runtime) + .readStream(shuffleReaderHandle, byteIn)) private var cb: ColumnarBatch = _ diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala index 37ea11a73d2a..b8e6513cf009 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala @@ -18,10 +18,8 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.memtarget.MemoryTarget -import org.apache.gluten.memory.memtarget.Spiller -import org.apache.gluten.memory.memtarget.Spillers -import org.apache.gluten.memory.nmm.NativeMemoryManagers +import org.apache.gluten.exec.Runtimes +import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.vectorized._ import org.apache.spark._ @@ -35,7 +33,6 @@ import org.apache.celeborn.client.ShuffleClient import org.apache.celeborn.common.CelebornConf import java.io.IOException -import java.util class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( shuffleId: Int, @@ -52,7 +49,9 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( client, writeMetrics) { - private val jniWrapper = ShuffleWriterJniWrapper.create() + private val runtime = Runtimes.contextInstance("CelebornShuffleWriter") + + private val jniWrapper = ShuffleWriterJniWrapper.create(runtime) private var splitResult: SplitResult = _ @@ -105,31 +104,6 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( clientPushBufferMaxSize, clientPushSortMemoryThreshold, celebornPartitionPusher, - NativeMemoryManagers - .create( - "CelebornShuffleWriter", - new Spiller() { - override def spill(self: MemoryTarget, size: Long): Long = { - if (nativeShuffleWriter == -1L) { - throw new IllegalStateException( - "Fatal: spill() called before a celeborn shuffle writer " + - "is created. This behavior should be" + - "optimized by moving memory " + - "allocations from make() to split()") - } - logInfo(s"Gluten shuffle writer: Trying to push $size bytes of data") - // fixme pass true when being called by self - val pushed = - jniWrapper.nativeEvict(nativeShuffleWriter, size, false) - logInfo(s"Gluten shuffle writer: Pushed $pushed / $size bytes of data") - pushed - } - - override def applicablePhases(): util.Set[Spiller.Phase] = - Spillers.PHASE_SET_SPILL_ONLY - } - ) - .getNativeInstanceHandle, handle, context.taskAttemptId(), GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, context.partitionId), @@ -137,6 +111,26 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( shuffleWriterType, GlutenConfig.getConf.columnarShuffleReallocThreshold ) + runtime.addSpiller(new Spiller() { + override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = { + if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { + return 0L + } + if (nativeShuffleWriter == -1L) { + throw new IllegalStateException( + "Fatal: spill() called before a celeborn shuffle writer " + + "is created. This behavior should be" + + "optimized by moving memory " + + "allocations from make() to split()") + } + logInfo(s"Gluten shuffle writer: Trying to push $size bytes of data") + // fixme pass true when being called by self + val pushed = + jniWrapper.nativeEvict(nativeShuffleWriter, size, false) + logInfo(s"Gluten shuffle writer: Pushed $pushed / $size bytes of data") + pushed + } + }) } val startTime = System.nanoTime() jniWrapper.write(nativeShuffleWriter, cb.numRows, handle, availableOffHeapPerTask()) diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java index c3ece743310a..75e3db2e7d1f 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java @@ -22,7 +22,6 @@ import org.apache.spark.memory.TaskMemoryManager; -import java.util.List; import java.util.Map; public final class MemoryTargets { @@ -54,7 +53,7 @@ public static MemoryTarget dynamicOffHeapSizingIfEnabled(MemoryTarget memoryTarg public static MemoryTarget newConsumer( TaskMemoryManager tmm, String name, - List spillers, + Spiller spiller, Map virtualChildren) { final TreeMemoryConsumers.Factory factory; if (GlutenConfig.getConf().memoryIsolation()) { @@ -63,6 +62,6 @@ public static MemoryTarget newConsumer( factory = TreeMemoryConsumers.shared(); } - return factory.newConsumer(tmm, name, spillers, virtualChildren); + return factory.newConsumer(tmm, name, spiller, virtualChildren); } } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java index 0e0457a7c7e7..a0ec6dc6c7c6 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java @@ -16,14 +16,11 @@ */ package org.apache.gluten.memory.memtarget; -import java.util.Set; - public interface Spiller { + long spill(MemoryTarget self, Phase phase, long size); - long spill(MemoryTarget self, long size); - - Set applicablePhases(); - + // Order of the elements matters, since + // consumer should call spillers with in the defined order. enum Phase { SHRINK, SPILL diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java index 6032b9494bc8..4477e2956db7 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java @@ -16,16 +16,21 @@ */ package org.apache.gluten.memory.memtarget; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.*; public final class Spillers { private Spillers() { // enclose factory ctor } + public static final Spiller NOOP = + new Spiller() { + @Override + public long spill(MemoryTarget self, Phase phase, long size) { + return 0; + } + }; + public static final Set PHASE_SET_ALL = Collections.unmodifiableSet( new HashSet<>(Arrays.asList(Spiller.Phase.SHRINK, Spiller.Phase.SPILL))); @@ -40,6 +45,10 @@ public static Spiller withMinSpillSize(Spiller spiller, long minSize) { return new WithMinSpillSize(spiller, minSize); } + public static AppendableSpillerList appendable() { + return new AppendableSpillerList(); + } + // Minimum spill target size should be larger than spark.gluten.memory.reservationBlockSize, // since any release action within size smaller than the block size may not have chance to // report back to the Java-side reservation listener. @@ -53,13 +62,27 @@ private WithMinSpillSize(Spiller delegated, long minSize) { } @Override - public long spill(MemoryTarget self, long size) { - return delegated.spill(self, Math.max(size, minSize)); + public long spill(MemoryTarget self, Spiller.Phase phase, long size) { + return delegated.spill(self, phase, Math.max(size, minSize)); + } + } + + public static class AppendableSpillerList implements Spiller { + private final List spillers = new LinkedList<>(); + + private AppendableSpillerList() {} + + public void append(Spiller spiller) { + spillers.add(spiller); } @Override - public Set applicablePhases() { - return delegated.applicablePhases(); + public long spill(MemoryTarget self, Phase phase, final long size) { + long remainingBytes = size; + for (Spiller spiller : spillers) { + remainingBytes -= spiller.spill(self, phase, remainingBytes); + } + return size - remainingBytes; } } } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java index 37d44fd7a675..9543bb5ed9eb 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java @@ -19,7 +19,6 @@ import org.apache.gluten.memory.MemoryUsageStatsBuilder; import org.apache.gluten.memory.memtarget.spark.TreeMemoryConsumer; -import java.util.List; import java.util.Map; /** An abstract for both {@link TreeMemoryConsumer} and it's non-consumer children nodes. */ @@ -29,12 +28,12 @@ public interface TreeMemoryTarget extends MemoryTarget, KnownNameAndStats { TreeMemoryTarget newChild( String name, long capacity, - List spillers, + Spiller spiller, Map virtualChildren); Map children(); TreeMemoryTarget parent(); - List getNodeSpillers(); + Spiller getNodeSpiller(); } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java index a8dc39ce3c16..24d9fc0e2d4a 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java @@ -23,19 +23,14 @@ import com.google.common.base.Preconditions; import org.apache.spark.util.Utils; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.PriorityQueue; import java.util.Queue; -import java.util.function.Predicate; import java.util.stream.Collectors; public class TreeMemoryTargets { - public static final List SPILL_PHASES = - Arrays.asList(Spiller.Phase.SHRINK, Spiller.Phase.SPILL); private TreeMemoryTargets() { // enclose factory ctor @@ -45,26 +40,24 @@ public static TreeMemoryTarget newChild( TreeMemoryTarget parent, String name, long capacity, - List spillers, + Spiller spiller, Map virtualChildren) { - return new Node(parent, name, capacity, spillers, virtualChildren); + return new Node(parent, name, capacity, spiller, virtualChildren); } public static long spillTree(TreeMemoryTarget node, final long bytes) { long remainingBytes = bytes; - for (Spiller.Phase phase : SPILL_PHASES) { + for (Spiller.Phase phase : Spiller.Phase.values()) { // First shrink, then if no good, spill. if (remainingBytes <= 0) { break; } - remainingBytes -= - spillTree(node, remainingBytes, spiller -> spiller.applicablePhases().contains(phase)); + remainingBytes -= spillTree(node, phase, remainingBytes); } return bytes - remainingBytes; } - private static long spillTree( - TreeMemoryTarget node, final long bytes, Predicate spillerFilter) { + private static long spillTree(TreeMemoryTarget node, Spiller.Phase phase, final long bytes) { // sort children by used bytes, descending Queue q = new PriorityQueue<>( @@ -83,13 +76,9 @@ private static long spillTree( if (remainingBytes > 0) { // if still doesn't fit, spill self - final List applicableSpillers = - node.getNodeSpillers().stream().filter(spillerFilter).collect(Collectors.toList()); - for (int i = 0; i < applicableSpillers.size() && remainingBytes > 0; i++) { - final Spiller spiller = applicableSpillers.get(i); - long spilled = spiller.spill(node, remainingBytes); - remainingBytes -= spilled; - } + final Spiller spiller = node.getNodeSpiller(); + long spilled = spiller.spill(node, phase, remainingBytes); + remainingBytes -= spilled; } return bytes - remainingBytes; @@ -101,7 +90,7 @@ public static class Node implements TreeMemoryTarget, KnownNameAndStats { private final TreeMemoryTarget parent; private final String name; private final long capacity; - private final List spillers; + private final Spiller spiller; private final Map virtualChildren; private final SimpleMemoryUsageRecorder selfRecorder = new SimpleMemoryUsageRecorder(); @@ -109,7 +98,7 @@ private Node( TreeMemoryTarget parent, String name, long capacity, - List spillers, + Spiller spiller, Map virtualChildren) { this.parent = parent; this.capacity = capacity; @@ -119,7 +108,7 @@ private Node( } else { this.name = String.format("%s, %s", uniqueName, Utils.bytesToString(capacity)); } - this.spillers = Collections.unmodifiableList(spillers); + this.spiller = spiller; this.virtualChildren = virtualChildren; } @@ -140,8 +129,8 @@ private long borrow0(long size) { } @Override - public List getNodeSpillers() { - return spillers; + public Spiller getNodeSpiller() { + return spiller; } private boolean ensureFreeCapacity(long bytesNeeded) { @@ -209,9 +198,9 @@ public MemoryUsageStats stats() { public TreeMemoryTarget newChild( String name, long capacity, - List spillers, + Spiller spiller, Map virtualChildren) { - final Node child = new Node(this, name, capacity, spillers, virtualChildren); + final Node child = new Node(this, name, capacity, spiller, virtualChildren); if (children.containsKey(child.name())) { throw new IllegalArgumentException("Child already registered: " + child.name()); } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java index 3028547c007a..76aa63aebb64 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java @@ -63,8 +63,16 @@ public RegularMemoryConsumer( } @Override - public long spill(long size, MemoryConsumer trigger) { - long spilledOut = spiller.spill(this, size); + public long spill(final long size, MemoryConsumer trigger) { + long remainingBytes = size; + for (Spiller.Phase phase : Spiller.Phase.values()) { + // First shrink, then if no good, spill. + if (remainingBytes <= 0) { + break; + } + remainingBytes -= spiller.spill(this, phase, size); + } + long spilledOut = size - remainingBytes; if (TaskResources.inSparkTask()) { TaskResources.getLocalTaskContext().taskMetrics().incMemoryBytesSpilled(spilledOut); } diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java index f86295b0697f..44c725798c75 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java @@ -18,11 +18,7 @@ import org.apache.gluten.memory.MemoryUsageStatsBuilder; import org.apache.gluten.memory.SimpleMemoryUsageRecorder; -import org.apache.gluten.memory.memtarget.MemoryTargetUtil; -import org.apache.gluten.memory.memtarget.MemoryTargetVisitor; -import org.apache.gluten.memory.memtarget.Spiller; -import org.apache.gluten.memory.memtarget.TreeMemoryTarget; -import org.apache.gluten.memory.memtarget.TreeMemoryTargets; +import org.apache.gluten.memory.memtarget.*; import org.apache.gluten.proto.MemoryUsageStats; import com.google.common.base.Preconditions; @@ -33,7 +29,6 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -124,10 +119,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException { public TreeMemoryTarget newChild( String name, long capacity, - List spillers, + Spiller spiller, Map virtualChildren) { final TreeMemoryTarget child = - TreeMemoryTargets.newChild(this, name, capacity, spillers, virtualChildren); + TreeMemoryTargets.newChild(this, name, capacity, spiller, virtualChildren); if (children.containsKey(child.name())) { throw new IllegalArgumentException("Child already registered: " + child.name()); } @@ -147,9 +142,9 @@ public TreeMemoryTarget parent() { } @Override - public List getNodeSpillers() { + public Spiller getNodeSpiller() { // root doesn't spill - return Collections.emptyList(); + return Spillers.NOOP; } public TaskMemoryManager getTaskMemoryManager() { diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java index 1da23d15e353..7ab05bd3a2e7 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java @@ -19,13 +19,13 @@ import org.apache.gluten.GlutenConfig; import org.apache.gluten.memory.MemoryUsageStatsBuilder; import org.apache.gluten.memory.memtarget.Spiller; +import org.apache.gluten.memory.memtarget.Spillers; import org.apache.gluten.memory.memtarget.TreeMemoryTarget; import org.apache.commons.collections.map.ReferenceMap; import org.apache.spark.memory.TaskMemoryManager; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -76,7 +76,7 @@ private TreeMemoryTarget getSharedAccount(TaskMemoryManager tmm) { m -> { TreeMemoryTarget tmc = new TreeMemoryConsumer((TaskMemoryManager) m); return tmc.newChild( - "root", perTaskCapacity, Collections.emptyList(), Collections.emptyMap()); + "root", perTaskCapacity, Spillers.NOOP, Collections.emptyMap()); }); } } @@ -84,11 +84,11 @@ private TreeMemoryTarget getSharedAccount(TaskMemoryManager tmm) { public TreeMemoryTarget newConsumer( TaskMemoryManager tmm, String name, - List spillers, + Spiller spiller, Map virtualChildren) { final TreeMemoryTarget account = getSharedAccount(tmm); return account.newChild( - name, TreeMemoryConsumer.CAPACITY_UNLIMITED, spillers, virtualChildren); + name, TreeMemoryConsumer.CAPACITY_UNLIMITED, spiller, virtualChildren); } } } diff --git a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala index 48ed08fb71ce..178310fd6497 100644 --- a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala @@ -44,6 +44,10 @@ object SparkMemoryUtil { consumersField.setAccessible(true) taskIdField.setAccessible(true) + def bytesToString(size: Long): String = { + Utils.bytesToString(size) + } + // We assume storage memory can be fully transferred to execution memory so far def getCurrentAvailableOffHeapMemory: Long = { val mm = SparkEnv.get.memoryManager diff --git a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java index e26765d33082..db018ffe4043 100644 --- a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java +++ b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java @@ -17,6 +17,7 @@ package org.apache.gluten.memory.memtarget.spark; import org.apache.gluten.GlutenConfig; +import org.apache.gluten.memory.memtarget.Spillers; import org.apache.gluten.memory.memtarget.TreeMemoryTarget; import org.apache.spark.TaskContext; @@ -43,7 +44,7 @@ public void testIsolated() { factory.newConsumer( TaskContext.get().taskMemoryManager(), "FOO", - Collections.emptyList(), + Spillers.NOOP, Collections.emptyMap()); Assert.assertEquals(20, consumer.borrow(20)); Assert.assertEquals(70, consumer.borrow(70)); @@ -65,7 +66,7 @@ public void testShared() { factory.newConsumer( TaskContext.get().taskMemoryManager(), "FOO", - Collections.emptyList(), + Spillers.NOOP, Collections.emptyMap()); Assert.assertEquals(20, consumer.borrow(20)); Assert.assertEquals(70, consumer.borrow(70)); @@ -87,7 +88,7 @@ public void testIsolatedAndShared() { .newConsumer( TaskContext.get().taskMemoryManager(), "FOO", - Collections.emptyList(), + Spillers.NOOP, Collections.emptyMap()); Assert.assertEquals(110, shared.borrow(110)); final TreeMemoryTarget isolated = @@ -95,7 +96,7 @@ public void testIsolatedAndShared() { .newConsumer( TaskContext.get().taskMemoryManager(), "FOO", - Collections.emptyList(), + Spillers.NOOP, Collections.emptyMap()); Assert.assertEquals(100, isolated.borrow(110)); }); diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java index 4f042b893832..a834e13a4348 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; public class ColumnarBatchJniWrapper implements RuntimeAware { private final Runtime runtime; @@ -27,11 +26,7 @@ private ColumnarBatchJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static ColumnarBatchJniWrapper create() { - return new ColumnarBatchJniWrapper(Runtimes.contextInstance()); - } - - public static ColumnarBatchJniWrapper forRuntime(Runtime runtime) { + public static ColumnarBatchJniWrapper create(Runtime runtime) { return new ColumnarBatchJniWrapper(runtime); } @@ -39,22 +34,21 @@ public static ColumnarBatchJniWrapper forRuntime(Runtime runtime) { public native long getForEmptySchema(int numRows); - public native String getType(long batchHandle); + public native String getType(long batch); - public native long numColumns(long batchHandle); + public native long numColumns(long batch); - public native long numRows(long batchHandle); + public native long numRows(long batch); - public native long numBytes(long batchHandle); + public native long numBytes(long batch); public native long compose(long[] batches); public native void exportToArrow(long batch, long cSchema, long cArray); - public native long select( - long nativeMemoryManagerHandle, // why a mm is needed here? - long batch, - int[] columnIndices); + public native long select(long batch, int[] columnIndices); + + public native long obtainOwnership(long batch); public native void close(long batch); diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java index e2cfa335d5c6..fc3b56c1bcad 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java @@ -19,13 +19,11 @@ import org.apache.gluten.exception.GlutenException; import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.Runtimes; -import org.apache.gluten.memory.nmm.NativeMemoryManager; import org.apache.gluten.utils.ArrowAbiUtil; import org.apache.gluten.utils.ArrowUtil; import org.apache.gluten.utils.ImplicitClass; import org.apache.gluten.vectorized.ArrowWritableColumnVector; -import com.google.common.base.Preconditions; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.CDataDictionaryProvider; @@ -38,11 +36,8 @@ import java.lang.reflect.Field; import java.util.Arrays; -import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.NoSuchElementException; -import java.util.Set; public class ColumnarBatches { private static final Field FIELD_COLUMNS; @@ -125,15 +120,14 @@ public static boolean isLightBatch(ColumnarBatch batch) { * This method will always return a velox based ColumnarBatch. This method will close the input * column batch. */ - public static ColumnarBatch select( - NativeMemoryManager nmm, ColumnarBatch batch, int[] columnIndices) { + public static ColumnarBatch select(ColumnarBatch batch, int[] columnIndices) { + final Runtime runtime = Runtimes.contextInstance("ColumnarBatches#select"); switch (identifyBatchType(batch)) { case LIGHT: final IndicatorVector iv = getIndicatorVector(batch); long outputBatchHandle = - ColumnarBatchJniWrapper.create() - .select(nmm.getNativeInstanceHandle(), iv.handle(), columnIndices); - return create(iv.runtime(), outputBatchHandle); + ColumnarBatchJniWrapper.create(runtime).select(iv.handle(), columnIndices); + return create(runtime, outputBatchHandle); case HEAVY: return new ColumnarBatch( Arrays.stream(columnIndices).mapToObj(batch::column).toArray(ColumnVector[]::new), @@ -181,7 +175,7 @@ private static ColumnarBatch load(BufferAllocator allocator, ColumnarBatch input ArrowArray cArray = ArrowArray.allocateNew(allocator); ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); CDataDictionaryProvider provider = new CDataDictionaryProvider()) { - ColumnarBatchJniWrapper.forRuntime(iv.runtime()) + ColumnarBatchJniWrapper.create(Runtimes.contextInstance("ColumnarBatches#load")) .exportToArrow(iv.handle(), cSchema.memoryAddress(), cArray.memoryAddress()); Data.exportSchema( @@ -217,12 +211,12 @@ private static ColumnarBatch offload(BufferAllocator allocator, ColumnarBatch in if (input.numCols() == 0) { throw new IllegalArgumentException("batch with zero columns cannot be offloaded"); } - final Runtime runtime = Runtimes.contextInstance(); + final Runtime runtime = Runtimes.contextInstance("ColumnarBatches#offload"); try (ArrowArray cArray = ArrowArray.allocateNew(allocator); ArrowSchema cSchema = ArrowSchema.allocateNew(allocator)) { ArrowAbiUtil.exportFromSparkColumnarBatch(allocator, input, cSchema, cArray); long handle = - ColumnarBatchJniWrapper.forRuntime(runtime) + ColumnarBatchJniWrapper.create(runtime) .createWithArrowArray(cSchema.memoryAddress(), cArray.memoryAddress()); ColumnarBatch output = ColumnarBatches.create(runtime, handle); @@ -333,18 +327,12 @@ public static long compose(ColumnarBatch... batches) { Arrays.stream(batches) .map(ColumnarBatches::getIndicatorVector) .toArray(IndicatorVector[]::new); - // We assume all input batches should be managed by same Runtime. - // FIXME: The check could be removed to adopt ownership-transfer semantic - final Runtime[] ctxs = - Arrays.stream(ivs).map(IndicatorVector::runtime).distinct().toArray(Runtime[]::new); - Preconditions.checkState( - ctxs.length == 1, "All input batches should be managed by same Runtime."); final long[] handles = Arrays.stream(ivs).mapToLong(IndicatorVector::handle).toArray(); - return ColumnarBatchJniWrapper.forRuntime(ctxs[0]).compose(handles); + return ColumnarBatchJniWrapper.create(Runtimes.contextInstance("ColumnarBatches#compose")) + .compose(handles); } - public static ColumnarBatch create(Runtime runtime, long nativeHandle) { - final IndicatorVector iv = new IndicatorVector(runtime, nativeHandle); + private static ColumnarBatch create(IndicatorVector iv) { int numColumns = Math.toIntExact(iv.getNumColumns()); int numRows = Math.toIntExact(iv.getNumRows()); if (numColumns == 0) { @@ -360,6 +348,10 @@ public static ColumnarBatch create(Runtime runtime, long nativeHandle) { return new ColumnarBatch(columnVectors, numRows); } + public static ColumnarBatch create(Runtime runtime, long nativeHandle) { + return create(new IndicatorVector(runtime, nativeHandle)); + } + public static void retain(ColumnarBatch b) { switch (identifyBatchType(b)) { case LIGHT: @@ -384,18 +376,4 @@ public static void release(ColumnarBatch b) { public static long getNativeHandle(ColumnarBatch batch) { return getIndicatorVector(batch).handle(); } - - public static Runtime getRuntime(ColumnarBatch batch) { - return getIndicatorVector(batch).runtime(); - } - - public static Runtime getRuntime(List batch) { - final Set all = new HashSet<>(); - batch.forEach(b -> all.add(getRuntime(b))); - if (all.size() != 1) { - throw new IllegalArgumentException( - "The input columnar batches has different associated runtimes"); - } - return all.toArray(new Runtime[0])[0]; - } } diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java index d15412bec7e5..0ec5b78ce500 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java @@ -18,146 +18,33 @@ import org.apache.gluten.exec.Runtime; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.Decimal; -import org.apache.spark.sql.vectorized.ColumnVector; -import org.apache.spark.sql.vectorized.ColumnarArray; -import org.apache.spark.sql.vectorized.ColumnarMap; -import org.apache.spark.unsafe.types.UTF8String; - import java.util.concurrent.atomic.AtomicLong; -public class IndicatorVector extends ColumnVector { - private final Runtime runtime; - private final long handle; +public class IndicatorVector extends IndicatorVectorBase { private final AtomicLong refCnt = new AtomicLong(1L); protected IndicatorVector(Runtime runtime, long handle) { - super(DataTypes.NullType); - this.runtime = runtime; - this.handle = handle; - } - - public Runtime runtime() { - return runtime; - } - - public String getType() { - return ColumnarBatchJniWrapper.forRuntime(runtime).getType(handle); - } - - public long getNumColumns() { - return ColumnarBatchJniWrapper.forRuntime(runtime).numColumns(handle); - } - - public long getNumRows() { - return ColumnarBatchJniWrapper.forRuntime(runtime).numRows(handle); + super(runtime, handle); } - public long refCnt() { + @Override + long refCnt() { return refCnt.get(); } - public void retain() { + @Override + void retain() { refCnt.getAndIncrement(); } @Override - public void close() { + void release() { if (refCnt.get() == 0) { // TODO use stronger restriction (IllegalStateException probably) return; } if (refCnt.decrementAndGet() == 0) { - ColumnarBatchJniWrapper.forRuntime(runtime).close(handle); + jniWrapper.close(handle); } } - - public boolean isClosed() { - return refCnt.get() == 0; - } - - @Override - public boolean hasNull() { - throw new UnsupportedOperationException(); - } - - @Override - public int numNulls() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isNullAt(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean getBoolean(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public byte getByte(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public short getShort(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public int getInt(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public long getLong(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public float getFloat(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public double getDouble(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnarArray getArray(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnarMap getMap(int ordinal) { - throw new UnsupportedOperationException(); - } - - @Override - public Decimal getDecimal(int rowId, int precision, int scale) { - throw new UnsupportedOperationException(); - } - - @Override - public UTF8String getUTF8String(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public byte[] getBinary(int rowId) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnVector getChild(int ordinal) { - throw new UnsupportedOperationException(); - } - - public long handle() { - return handle; - } } diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java new file mode 100644 index 000000000000..fa695127adbf --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.columnarbatch; + +import org.apache.gluten.exec.Runtime; + +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarArray; +import org.apache.spark.sql.vectorized.ColumnarMap; +import org.apache.spark.unsafe.types.UTF8String; + +public abstract class IndicatorVectorBase extends ColumnVector { + private final Runtime runtime; + protected final long handle; + protected final ColumnarBatchJniWrapper jniWrapper; + + protected IndicatorVectorBase(Runtime runtime, long handle) { + super(DataTypes.NullType); + this.runtime = runtime; + this.jniWrapper = ColumnarBatchJniWrapper.create(runtime); + this.handle = takeOwnership(handle); + } + + private long takeOwnership(long handle) { + // Note: Underlying memory of returned batch still holds + // reference to the original memory manager. As + // a result, once its original resident runtime / mm is + // released, data may become invalid. Currently, it's + // the caller's responsibility to make sure the original + // runtime / mm keep alive even this function + // was called. + // + // Additionally, as in Gluten we have principle that runtime + // mm that were created earlier will be released + // later, this FILO practice is what helps the runtime that + // took ownership be able to access the data constantly + // because the original runtime will live longer than + // itself. + long newHandle = jniWrapper.obtainOwnership(handle); + jniWrapper.close(handle); + return newHandle; + } + + public String getType() { + return jniWrapper.getType(handle); + } + + public long getNumColumns() { + return jniWrapper.numColumns(handle); + } + + public long getNumRows() { + return jniWrapper.numRows(handle); + } + + abstract long refCnt(); + + abstract void retain(); + + abstract void release(); + + @Override + public void close() { + release(); + } + + @Override + public boolean hasNull() { + throw new UnsupportedOperationException(); + } + + @Override + public int numNulls() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isNullAt(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getBoolean(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public byte getByte(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public short getShort(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public int getInt(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public long getLong(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public float getFloat(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public double getDouble(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnarArray getArray(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnarMap getMap(int ordinal) { + throw new UnsupportedOperationException(); + } + + @Override + public Decimal getDecimal(int rowId, int precision, int scale) { + throw new UnsupportedOperationException(); + } + + @Override + public UTF8String getUTF8String(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getBinary(int rowId) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnVector getChild(int ordinal) { + throw new UnsupportedOperationException(); + } + + public long handle() { + return handle; + } +} diff --git a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java index 46d12c1deee9..5a34196c4eb7 100644 --- a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; import org.apache.gluten.init.JniUtils; import org.apache.gluten.vectorized.ColumnarBatchInIterator; @@ -35,8 +34,8 @@ private DatasourceJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static DatasourceJniWrapper create() { - return new DatasourceJniWrapper(Runtimes.contextInstance()); + public static DatasourceJniWrapper create(Runtime runtime) { + return new DatasourceJniWrapper(runtime); } @Override @@ -44,14 +43,11 @@ public long handle() { return runtime.getHandle(); } - public long nativeInitDatasource( - String filePath, long cSchema, long memoryManagerHandle, Map options) { - return nativeInitDatasource( - filePath, cSchema, memoryManagerHandle, JniUtils.toNativeConf(options)); + public long nativeInitDatasource(String filePath, long cSchema, Map options) { + return nativeInitDatasource(filePath, cSchema, JniUtils.toNativeConf(options)); } - public native long nativeInitDatasource( - String filePath, long cSchema, long memoryManagerHandle, byte[] options); + public native long nativeInitDatasource(String filePath, long cSchema, byte[] options); public native void inspectSchema(long dsHandle, long cSchemaAddress); @@ -60,5 +56,5 @@ public native long nativeInitDatasource( public native void write(long dsHandle, ColumnarBatchInIterator iterator); public native BlockStripes splitBlockByPartitionAndBucket( - long blockAddress, int[] partitionColIndice, boolean hasBucket, long memoryManagerId); + long blockAddress, int[] partitionColIndice, boolean hasBucket); } diff --git a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java index 14ec87dcf6ca..d2a18e9b4930 100644 --- a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java @@ -16,11 +16,21 @@ */ package org.apache.gluten.exec; +import org.apache.gluten.memory.listener.ReservationListener; + public class RuntimeJniWrapper { private RuntimeJniWrapper() {} - public static native long createRuntime(String backendType, byte[] sessionConf); + public static native long createRuntime( + String backendType, ReservationListener listener, byte[] sessionConf); + + // Memory management. + public static native byte[] collectMemoryUsage(long handle); + + public static native long shrinkMemory(long handle, long size); + + public static native void holdMemory(long handle); public static native void releaseRuntime(long handle); } diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java b/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java deleted file mode 100644 index 6860d68e3c1b..000000000000 --- a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.memory.alloc; - -/** - * This along with {@link NativeMemoryAllocators}, as built-in toolkit for managing native memory - * allocations. - */ -public class NativeMemoryAllocator { - enum Type { - DEFAULT, - } - - private final long nativeInstanceId; - - public NativeMemoryAllocator(long nativeInstanceId) { - this.nativeInstanceId = nativeInstanceId; - } - - public static NativeMemoryAllocator create(Type type) { - return new NativeMemoryAllocator(getAllocator(type.name())); - } - - public long getNativeInstanceId() { - return this.nativeInstanceId; - } - - public void close() { - releaseAllocator(this.nativeInstanceId); - } - - private static native long getAllocator(String typeName); - - private static native void releaseAllocator(long allocatorId); -} diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java deleted file mode 100644 index e60766ac1f95..000000000000 --- a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.memory.alloc; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Built-in toolkit for managing native memory allocations. To use the facility, one should import - * Gluten's C++ library then create the c++ instance using following example code: - * - *

```c++ auto* allocator = reinterpret_cast(allocator_id); ``` - * - *

The ID "allocator_id" can be retrieved from Java API {@link - * NativeMemoryAllocator#getNativeInstanceId()}. - * - *

FIXME: to export the native APIs in a standard way - */ -public final class NativeMemoryAllocators { - private static final Map INSTANCES = - new ConcurrentHashMap<>(); - - private final NativeMemoryAllocator allocator; - - private NativeMemoryAllocators(NativeMemoryAllocator.Type type) { - allocator = NativeMemoryAllocator.create(type); - } - - public static NativeMemoryAllocators getDefault() { - return forType(NativeMemoryAllocator.Type.DEFAULT); - } - - private static NativeMemoryAllocators forType(NativeMemoryAllocator.Type type) { - return INSTANCES.computeIfAbsent(type, NativeMemoryAllocators::new); - } - - public NativeMemoryAllocator get() { - return allocator; - } -} diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java index 51f49da704eb..f51852ab8d2b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java @@ -17,6 +17,7 @@ package org.apache.gluten.memory.arrow.alloc; import org.apache.gluten.memory.memtarget.MemoryTargets; +import org.apache.gluten.memory.memtarget.Spillers; import org.apache.arrow.memory.AllocationListener; import org.apache.arrow.memory.BufferAllocator; @@ -62,10 +63,7 @@ public static class ArrowBufferAllocatorManager implements TaskResource { MemoryTargets.throwOnOom( MemoryTargets.dynamicOffHeapSizingIfEnabled( MemoryTargets.newConsumer( - tmm, - "ArrowContextInstance", - Collections.emptyList(), - Collections.emptyMap()))), + tmm, "ArrowContextInstance", Spillers.NOOP, Collections.emptyMap()))), TaskResources.getSharedUsage()); } diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ManagedReservationListener.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ManagedReservationListener.java similarity index 98% rename from gluten-data/src/main/java/org/apache/gluten/memory/nmm/ManagedReservationListener.java rename to gluten-data/src/main/java/org/apache/gluten/memory/listener/ManagedReservationListener.java index 88e793320920..b7d6ecd67589 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ManagedReservationListener.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ManagedReservationListener.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.memory.nmm; +package org.apache.gluten.memory.listener; import org.apache.gluten.memory.SimpleMemoryUsageRecorder; import org.apache.gluten.memory.memtarget.MemoryTarget; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ReservationListener.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListener.java similarity index 76% rename from gluten-data/src/main/java/org/apache/gluten/memory/nmm/ReservationListener.java rename to gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListener.java index f423bc8fb4e4..6f8cb867e003 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ReservationListener.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListener.java @@ -14,14 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.memory.nmm; - -import org.apache.gluten.memory.SimpleMemoryUsageRecorder; -import org.apache.gluten.memory.memtarget.NoopMemoryTarget; +package org.apache.gluten.memory.listener; public interface ReservationListener { - ReservationListener NOOP = - new ManagedReservationListener(new NoopMemoryTarget(), new SimpleMemoryUsageRecorder()); long reserve(long size); diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java new file mode 100644 index 000000000000..47b9937eb7a3 --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.memory.listener; + +import org.apache.gluten.GlutenConfig; +import org.apache.gluten.memory.MemoryUsageStatsBuilder; +import org.apache.gluten.memory.SimpleMemoryUsageRecorder; +import org.apache.gluten.memory.memtarget.*; + +import org.apache.spark.memory.TaskMemoryManager; +import org.apache.spark.util.TaskResources; + +import java.util.Collections; +import java.util.Map; + +public final class ReservationListeners { + public static final ReservationListener NOOP = + new ManagedReservationListener(new NoopMemoryTarget(), new SimpleMemoryUsageRecorder()); + + public static ReservationListener create( + String name, Spiller spiller, Map mutableStats) { + if (!TaskResources.inSparkTask()) { + throw new IllegalStateException( + "Spillable reservation listener must be used in a Spark task."); + } + return create0(name, spiller, mutableStats); + } + + private static ReservationListener create0( + String name, Spiller spiller, Map mutableStats) { + // Memory target. + final double overAcquiredRatio = GlutenConfig.getConf().memoryOverAcquiredRatio(); + final long reservationBlockSize = GlutenConfig.getConf().memoryReservationBlockSize(); + final TaskMemoryManager tmm = TaskResources.getLocalTaskContext().taskMemoryManager(); + final MemoryTarget target = + MemoryTargets.throwOnOom( + MemoryTargets.overAcquire( + MemoryTargets.dynamicOffHeapSizingIfEnabled( + MemoryTargets.newConsumer( + tmm, + name, + Spillers.withMinSpillSize(spiller, reservationBlockSize), + mutableStats)), + MemoryTargets.dynamicOffHeapSizingIfEnabled( + MemoryTargets.newConsumer( + tmm, + "OverAcquire.DummyTarget", + new Spiller() { + @Override + public long spill(MemoryTarget self, Spiller.Phase phase, long size) { + if (!Spillers.PHASE_SET_ALL.contains(phase)) { + return 0L; + } + return self.repay(size); + } + }, + Collections.emptyMap())), + overAcquiredRatio)); + + // Listener. + return new ManagedReservationListener(target, TaskResources.getSharedUsage()); + } +} diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java deleted file mode 100644 index 0d1a0c5aec41..000000000000 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.memory.nmm; - -import org.apache.gluten.backendsapi.BackendsApiManager; -import org.apache.gluten.memory.alloc.NativeMemoryAllocators; -import org.apache.gluten.memory.memtarget.KnownNameAndStats; -import org.apache.gluten.proto.MemoryUsageStats; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.spark.memory.SparkMemoryUtil; -import org.apache.spark.util.TaskResource; -import org.apache.spark.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class NativeMemoryManager implements TaskResource { - - private static final Logger LOGGER = LoggerFactory.getLogger(NativeMemoryManager.class); - - private final long nativeInstanceHandle; - private final String name; - private final ReservationListener listener; - - private NativeMemoryManager( - String name, long nativeInstanceHandle, ReservationListener listener) { - this.name = name; - this.nativeInstanceHandle = nativeInstanceHandle; - this.listener = listener; - } - - public static NativeMemoryManager create(String name, ReservationListener listener) { - long allocatorId = NativeMemoryAllocators.getDefault().get().getNativeInstanceId(); - return new NativeMemoryManager( - name, create(BackendsApiManager.getBackendName(), name, allocatorId, listener), listener); - } - - public long getNativeInstanceHandle() { - return this.nativeInstanceHandle; - } - - public MemoryUsageStats collectMemoryUsage() { - try { - return MemoryUsageStats.parseFrom(collectMemoryUsage(nativeInstanceHandle)); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - public long shrink(long size) { - return shrink(nativeInstanceHandle, size); - } - - // Hold this memory manager. The underlying memory pools will be released as lately as this - // memory manager gets destroyed. Which means, a call to this function would make sure the - // memory blocks directly or indirectly managed by this manager, be guaranteed safe to - // access during the period that this manager is alive. - public void hold() { - hold(nativeInstanceHandle); - } - - private static native long shrink(long memoryManagerId, long size); - - private static native long create( - String backendType, String name, long allocatorId, ReservationListener listener); - - private static native void release(long memoryManagerId); - - private static native byte[] collectMemoryUsage(long memoryManagerId); - - private static native void hold(long memoryManagerId); - - @Override - public void release() throws Exception { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - SparkMemoryUtil.prettyPrintStats( - "About to release memory manager, usage dump:", - new KnownNameAndStats() { - @Override - public String name() { - return name; - } - - @Override - public MemoryUsageStats stats() { - return collectMemoryUsage(); - } - })); - } - release(nativeInstanceHandle); - if (listener.getUsedBytes() != 0) { - LOGGER.warn( - String.format( - "%s Reservation listener %s still reserved non-zero bytes, " - + "which may cause memory leak, size: %s. ", - name, listener.toString(), Utils.bytesToString(listener.getUsedBytes()))); - } - } - - @Override - public int priority() { - return 0; // lowest release priority - } - - @Override - public String resourceName() { - return name + "_mem"; - } -} diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java deleted file mode 100644 index 37456badd42f..000000000000 --- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.memory.nmm; - -import org.apache.gluten.GlutenConfig; -import org.apache.gluten.memory.MemoryUsageRecorder; -import org.apache.gluten.memory.memtarget.MemoryTarget; -import org.apache.gluten.memory.memtarget.MemoryTargets; -import org.apache.gluten.memory.memtarget.Spiller; -import org.apache.gluten.memory.memtarget.Spillers; -import org.apache.gluten.proto.MemoryUsageStats; - -import org.apache.spark.memory.TaskMemoryManager; -import org.apache.spark.util.TaskResources; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public final class NativeMemoryManagers { - private static final Logger LOG = LoggerFactory.getLogger(NativeMemoryManagers.class); - - // TODO: Let all caller support spill. - public static NativeMemoryManager contextInstance(String name) { - if (!TaskResources.inSparkTask()) { - throw new IllegalStateException("This method must be called in a Spark task."); - } - String id = "NativeMemoryManager:" + name; - return TaskResources.addResourceIfNotRegistered( - id, () -> createNativeMemoryManager(name, Collections.emptyList())); - } - - public static NativeMemoryManager create(String name, Spiller... spillers) { - if (!TaskResources.inSparkTask()) { - throw new IllegalStateException("Spiller must be used in a Spark task."); - } - - final NativeMemoryManager manager = createNativeMemoryManager(name, Arrays.asList(spillers)); - return TaskResources.addAnonymousResource(manager); - } - - private static NativeMemoryManager createNativeMemoryManager( - String name, List spillers) { - final AtomicReference out = new AtomicReference<>(); - // memory target - final double overAcquiredRatio = GlutenConfig.getConf().memoryOverAcquiredRatio(); - final long reservationBlockSize = GlutenConfig.getConf().memoryReservationBlockSize(); - final TaskMemoryManager tmm = TaskResources.getLocalTaskContext().taskMemoryManager(); - final MemoryTarget target = - MemoryTargets.throwOnOom( - MemoryTargets.overAcquire( - MemoryTargets.dynamicOffHeapSizingIfEnabled( - MemoryTargets.newConsumer( - tmm, - name, - // call memory manager's shrink API, if no good then call the spiller - Stream.concat( - Stream.of( - new Spiller() { - @Override - public long spill(MemoryTarget self, long size) { - return Optional.ofNullable(out.get()) - .map(nmm -> nmm.shrink(size)) - .orElseGet( - () -> { - LOG.warn( - "Shrink is requested before native " - + "memory manager is created. Try moving " - + "any actions about memory allocation" - + " out from the memory manager" - + " constructor."); - return 0L; - }); - } - - @Override - public Set applicablePhases() { - return Spillers.PHASE_SET_SHRINK_ONLY; - } - }), - spillers.stream()) - .map( - spiller -> Spillers.withMinSpillSize(spiller, reservationBlockSize)) - .collect(Collectors.toList()), - Collections.singletonMap( - "single", - new MemoryUsageRecorder() { - @Override - public void inc(long bytes) { - // no-op - } - - @Override - public long peak() { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public long current() { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public MemoryUsageStats toStats() { - return getNativeMemoryManager().collectMemoryUsage(); - } - - private NativeMemoryManager getNativeMemoryManager() { - return Optional.ofNullable(out.get()) - .orElseThrow( - () -> - new IllegalStateException( - "" - + "Memory usage stats are requested before" - + " native memory manager is created. Try" - + " moving any actions about memory" - + " allocation out from the memory manager" - + " constructor.")); - } - }))), - MemoryTargets.dynamicOffHeapSizingIfEnabled( - MemoryTargets.newConsumer( - tmm, - "OverAcquire.DummyTarget", - Collections.singletonList( - new Spiller() { - @Override - public long spill(MemoryTarget self, long size) { - return self.repay(size); - } - - @Override - public Set applicablePhases() { - return Spillers.PHASE_SET_ALL; - } - }), - Collections.emptyMap())), - overAcquiredRatio)); - // listener - ManagedReservationListener rl = - new ManagedReservationListener(target, TaskResources.getSharedUsage()); - // native memory manager - out.set(NativeMemoryManager.create(name, rl)); - return out.get(); - } -} diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java index bd89f62a1806..c147862d0139 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java @@ -18,6 +18,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatchJniWrapper; import org.apache.gluten.columnarbatch.ColumnarBatches; +import org.apache.gluten.exec.Runtimes; import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -25,6 +26,7 @@ import java.util.Iterator; public class ColumnarBatchInIterator extends GeneralInIterator { + public ColumnarBatchInIterator(Iterator delegated) { super(delegated); } @@ -33,7 +35,8 @@ public long next() { final ColumnarBatch next = nextColumnarBatch(); if (next.numCols() == 0) { // the operation will find a zero column batch from a task-local pool - return ColumnarBatchJniWrapper.create().getForEmptySchema(next.numRows()); + return ColumnarBatchJniWrapper.create(Runtimes.contextInstance("ColumnarBatchInIterator")) + .getForEmptySchema(next.numRows()); } final ColumnarBatch offloaded = ColumnarBatches.ensureOffloaded(ArrowBufferAllocators.contextInstance(), next); diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index 3a2a741bef0b..ddf00844f9b0 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -19,7 +19,6 @@ import org.apache.gluten.columnarbatch.ColumnarBatches; import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.memory.nmm.NativeMemoryManager; import org.apache.gluten.metrics.IMetrics; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -29,13 +28,11 @@ public class ColumnarBatchOutIterator extends GeneralOutIterator implements RuntimeAware { private final Runtime runtime; private final long iterHandle; - private final NativeMemoryManager nmm; - public ColumnarBatchOutIterator(Runtime runtime, long iterHandle, NativeMemoryManager nmm) { + public ColumnarBatchOutIterator(Runtime runtime, long iterHandle) { super(); this.runtime = runtime; this.iterHandle = iterHandle; - this.nmm = nmm; } @Override @@ -88,8 +85,9 @@ public long spill(long size) { @Override public void closeInternal() { - nmm.hold(); // to make sure the outputted batches are still accessible after the iterator is - // closed + // To make sure the outputted batches are still accessible after the iterator is closed. + // TODO: Remove this API if we have other choice, e.g., hold the pools in native code. + runtime.holdMemory(); nativeClose(iterHandle); } } diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java index 59f31689e107..bfe0d756112f 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; public class ColumnarBatchSerializerJniWrapper implements RuntimeAware { private final Runtime runtime; @@ -27,11 +26,7 @@ private ColumnarBatchSerializerJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static ColumnarBatchSerializerJniWrapper create() { - return new ColumnarBatchSerializerJniWrapper(Runtimes.contextInstance()); - } - - public static ColumnarBatchSerializerJniWrapper forRuntime(Runtime runtime) { + public static ColumnarBatchSerializerJniWrapper create(Runtime runtime) { return new ColumnarBatchSerializerJniWrapper(runtime); } @@ -40,10 +35,10 @@ public long handle() { return runtime.getHandle(); } - public native ColumnarBatchSerializeResult serialize(long[] handles, long memoryManagerHandle); + public native ColumnarBatchSerializeResult serialize(long[] handles); // Return the native ColumnarBatchSerializer handle - public native long init(long cSchema, long memoryManagerHandle); + public native long init(long cSchema); public native long deserialize(long serializerHandle, byte[] data); diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java index 7ae2fafb31b4..7f8de78f95ef 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; public class NativeColumnarToRowJniWrapper implements RuntimeAware { private final Runtime runtime; @@ -27,8 +26,8 @@ private NativeColumnarToRowJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static NativeColumnarToRowJniWrapper create() { - return new NativeColumnarToRowJniWrapper(Runtimes.contextInstance()); + public static NativeColumnarToRowJniWrapper create(Runtime runtime) { + return new NativeColumnarToRowJniWrapper(runtime); } @Override @@ -36,9 +35,9 @@ public long handle() { return runtime.getHandle(); } - public native long nativeColumnarToRowInit(long memoryManagerHandle) throws RuntimeException; + public native long nativeColumnarToRowInit() throws RuntimeException; - public native NativeColumnarToRowInfo nativeColumnarToRowConvert(long batchHandle, long c2rHandle) + public native NativeColumnarToRowInfo nativeColumnarToRowConvert(long c2rHandle, long batchHandle) throws RuntimeException; public native void nativeClose(long c2rHandle); diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java index 2ac048b2b960..e5eea029b2b3 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java @@ -22,8 +22,6 @@ import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; -import org.apache.gluten.memory.nmm.NativeMemoryManager; -import org.apache.gluten.memory.nmm.NativeMemoryManagers; import org.apache.gluten.utils.DebugUtil; import org.apache.gluten.validate.NativePlanValidationInfo; @@ -33,17 +31,15 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Optional; -import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; public class NativePlanEvaluator { + private final Runtime runtime = Runtimes.contextInstance("WholeStageIterator"); private final PlanEvaluatorJniWrapper jniWrapper; private NativePlanEvaluator() { - jniWrapper = PlanEvaluatorJniWrapper.create(); + jniWrapper = PlanEvaluatorJniWrapper.create(runtime); } public static NativePlanEvaluator create() { @@ -63,31 +59,6 @@ public void injectWriteFilesTempPath(String path) { public GeneralOutIterator createKernelWithBatchIterator( byte[] wsPlan, byte[][] splitInfo, List iterList, int partitionIndex) throws RuntimeException, IOException { - final AtomicReference outIterator = new AtomicReference<>(); - final NativeMemoryManager nmm = - NativeMemoryManagers.create( - "WholeStageIterator", - new Spiller() { - @Override - public long spill(MemoryTarget self, long size) { - ColumnarBatchOutIterator instance = - Optional.ofNullable(outIterator.get()) - .orElseThrow( - () -> - new IllegalStateException( - "Fatal: spill() called before a output iterator " - + "is created. This behavior should be optimized " - + "by moving memory allocations from create() to " - + "hasNext()/next()")); - return instance.spill(size); - } - - @Override - public Set applicablePhases() { - return Spillers.PHASE_SET_SPILL_ONLY; - } - }); - final long memoryManagerHandle = nmm.getNativeInstanceHandle(); final String spillDirPath = SparkDirectoryUtil.get() @@ -95,9 +66,8 @@ public Set applicablePhases() { .mkChildDirRoundRobin(UUID.randomUUID().toString()) .getAbsolutePath(); - long iterHandle = + final long itrHandle = jniWrapper.nativeCreateKernelWithIterator( - memoryManagerHandle, wsPlan, splitInfo, iterList.toArray(new GeneralInIterator[0]), @@ -106,12 +76,21 @@ public Set applicablePhases() { TaskContext.get().taskAttemptId(), DebugUtil.saveInputToFile(), BackendsApiManager.getSparkPlanExecApiInstance().rewriteSpillPath(spillDirPath)); - outIterator.set(createOutIterator(Runtimes.contextInstance(), iterHandle, nmm)); - return outIterator.get(); + final ColumnarBatchOutIterator out = createOutIterator(runtime, itrHandle); + runtime.addSpiller( + new Spiller() { + @Override + public long spill(MemoryTarget self, Spiller.Phase phase, long size) { + if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { + return 0L; + } + return out.spill(size); + } + }); + return out; } - private ColumnarBatchOutIterator createOutIterator( - Runtime runtime, long iterHandle, NativeMemoryManager nmm) throws IOException { - return new ColumnarBatchOutIterator(runtime, iterHandle, nmm); + private ColumnarBatchOutIterator createOutIterator(Runtime runtime, long itrHandle) { + return new ColumnarBatchOutIterator(runtime, itrHandle); } } diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java index b43c92b99142..1185d52fee77 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; public class NativeRowToColumnarJniWrapper implements RuntimeAware { private final Runtime runtime; @@ -27,8 +26,8 @@ private NativeRowToColumnarJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static NativeRowToColumnarJniWrapper create() { - return new NativeRowToColumnarJniWrapper(Runtimes.contextInstance()); + public static NativeRowToColumnarJniWrapper create(Runtime runtime) { + return new NativeRowToColumnarJniWrapper(runtime); } @Override @@ -36,7 +35,7 @@ public long handle() { return runtime.getHandle(); } - public native long init(long cSchema, long memoryManagerHandle); + public native long init(long cSchema); public native long nativeConvertRowToColumnar( long r2cHandle, long[] rowLength, long bufferAddress); diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java index 5e8ae392af8d..eecd7c9e2879 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; import org.apache.gluten.validate.NativePlanValidationInfo; /** @@ -33,8 +32,8 @@ private PlanEvaluatorJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static PlanEvaluatorJniWrapper create() { - return new PlanEvaluatorJniWrapper(Runtimes.contextInstance()); + public static PlanEvaluatorJniWrapper create(Runtime runtime) { + return new PlanEvaluatorJniWrapper(runtime); } @Override @@ -57,11 +56,9 @@ public long handle() { /** * Create a native compute kernel and return a columnar result iterator. * - * @param memoryManagerHandle NativeMemoryManager instance handle * @return iterator instance id */ public native long nativeCreateKernelWithIterator( - long memoryManagerHandle, byte[] wsPlan, byte[][] splitInfo, GeneralInIterator[] batchItr, diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java index 24425ccf72e6..515486e45a5b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; public class ShuffleReaderJniWrapper implements RuntimeAware { private final Runtime runtime; @@ -27,8 +26,8 @@ private ShuffleReaderJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static ShuffleReaderJniWrapper create() { - return new ShuffleReaderJniWrapper(Runtimes.contextInstance()); + public static ShuffleReaderJniWrapper create(Runtime runtime) { + return new ShuffleReaderJniWrapper(runtime); } @Override @@ -38,7 +37,6 @@ public long handle() { public native long make( long cSchema, - long memoryManagerHandle, String compressionType, String compressionCodecBackend, int batchSize, diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java index f4e1172757fe..883fc600171f 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java @@ -18,7 +18,6 @@ import org.apache.gluten.exec.Runtime; import org.apache.gluten.exec.RuntimeAware; -import org.apache.gluten.exec.Runtimes; import java.io.IOException; @@ -29,8 +28,8 @@ private ShuffleWriterJniWrapper(Runtime runtime) { this.runtime = runtime; } - public static ShuffleWriterJniWrapper create() { - return new ShuffleWriterJniWrapper(Runtimes.contextInstance()); + public static ShuffleWriterJniWrapper create(Runtime runtime) { + return new ShuffleWriterJniWrapper(runtime); } @Override @@ -65,7 +64,6 @@ public long make( String dataFile, int subDirsPerLocalDir, String localDirs, - long memoryManagerHandle, double reallocThreshold, long handle, long taskAttemptId, @@ -85,7 +83,6 @@ public long make( dataFile, subDirsPerLocalDir, localDirs, - memoryManagerHandle, reallocThreshold, handle, taskAttemptId, @@ -115,7 +112,6 @@ public long makeForRSS( int pushBufferMaxSize, long sortBufferMaxSize, Object pusher, - long memoryManagerHandle, long handle, long taskAttemptId, int startPartitionId, @@ -136,7 +132,6 @@ public long makeForRSS( null, 0, null, - memoryManagerHandle, reallocThreshold, handle, taskAttemptId, @@ -162,7 +157,6 @@ public native long nativeMake( String dataFile, int subDirsPerLocalDir, String localDirs, - long memoryManagerHandle, double reallocThreshold, long handle, long taskAttemptId, diff --git a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala index 12d855c7145a..1f632659eadf 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala @@ -20,35 +20,111 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenException import org.apache.gluten.init.JniUtils +import org.apache.gluten.memory.MemoryUsageStatsBuilder +import org.apache.gluten.memory.listener.ReservationListeners +import org.apache.gluten.memory.memtarget.{KnownNameAndStats, MemoryTarget, Spiller, Spillers} +import org.apache.gluten.proto.MemoryUsageStats -import org.apache.spark.sql.internal.GlutenConfigUtil -import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.memory.SparkMemoryUtil +import org.apache.spark.sql.internal.{GlutenConfigUtil, SQLConf} import org.apache.spark.util.TaskResource +import org.slf4j.LoggerFactory + import java.util.concurrent.atomic.AtomicBoolean -class Runtime private[exec] () extends TaskResource { - private val handle = RuntimeJniWrapper.createRuntime( - BackendsApiManager.getBackendName, - JniUtils.toNativeConf( - GlutenConfig.getNativeSessionConf( - BackendsApiManager.getSettings.getBackendConfigPrefix, - GlutenConfigUtil.parseConfig(SQLConf.get.getAllConfs))) - ) +import scala.collection.JavaConverters._ +import scala.collection.mutable + +trait Runtime { + def addSpiller(spiller: Spiller): Unit + def holdMemory(): Unit + def collectMemoryUsage(): MemoryUsageStats + def getHandle(): Long +} + +object Runtime { + private[exec] def apply(name: String): Runtime with TaskResource = { + new RuntimeImpl(name) + } + + private class RuntimeImpl(name: String) extends Runtime with TaskResource { + private val LOGGER = LoggerFactory.getLogger(classOf[Runtime]) + + private val spillers = Spillers.appendable() + private val mutableStats: mutable.Map[String, MemoryUsageStatsBuilder] = mutable.Map() + private val rl = ReservationListeners.create(resourceName(), spillers, mutableStats.asJava) + private val handle = RuntimeJniWrapper.createRuntime( + BackendsApiManager.getBackendName, + rl, + JniUtils.toNativeConf( + GlutenConfig.getNativeSessionConf( + BackendsApiManager.getSettings.getBackendConfigPrefix, + GlutenConfigUtil.parseConfig(SQLConf.get.getAllConfs))) + ) - private val released: AtomicBoolean = new AtomicBoolean(false) + spillers.append(new Spiller() { + override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = { + if (!Spillers.PHASE_SET_SHRINK_ONLY.contains(phase)) { + // Only respond for shrinking. + return 0L + } + RuntimeJniWrapper.shrinkMemory(handle, size) + } + }) + mutableStats += "single" -> new MemoryUsageStatsBuilder { + override def toStats: MemoryUsageStats = collectMemoryUsage() + } - def getHandle: Long = handle + private val released: AtomicBoolean = new AtomicBoolean(false) - override def release(): Unit = { - if (!released.compareAndSet(false, true)) { - throw new GlutenException( - s"Runtime instance already released: $getHandle, ${resourceName()}, ${priority()}") + def getHandle: Long = handle + + def addSpiller(spiller: Spiller): Unit = { + spillers.append(spiller) } - RuntimeJniWrapper.releaseRuntime(handle) - } - override def priority(): Int = 10 + def holdMemory(): Unit = { + RuntimeJniWrapper.holdMemory(handle) + } + + def collectMemoryUsage(): MemoryUsageStats = { + MemoryUsageStats.parseFrom(RuntimeJniWrapper.collectMemoryUsage(handle)) + } + + override def release(): Unit = { + if (!released.compareAndSet(false, true)) { + throw new GlutenException( + s"Runtime instance already released: $handle, ${resourceName()}, ${priority()}") + } + if (LOGGER.isDebugEnabled) { + LOGGER.debug( + SparkMemoryUtil.prettyPrintStats( + "About to release memory manager, usage dump:", + new KnownNameAndStats() { + override def name: String = resourceName() + + override def stats: MemoryUsageStats = collectMemoryUsage() + } + )) + } - override def resourceName(): String = s"Runtime_" + handle + RuntimeJniWrapper.releaseRuntime(handle) + + if (rl.getUsedBytes != 0) { + LOGGER.warn( + String.format( + "%s Reservation listener %s still reserved non-zero bytes, which may cause memory" + + " leak, size: %s. ", + name, + rl.toString, + SparkMemoryUtil.bytesToString(rl.getUsedBytes) + )) + } + } + + override def priority(): Int = 0 + + override def resourceName(): String = name + } } diff --git a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala index e5204771ce52..3614fe05f6b6 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala @@ -16,21 +16,20 @@ */ package org.apache.gluten.exec -import org.apache.spark.util.TaskResources +import org.apache.spark.util.{TaskResource, TaskResources} object Runtimes { - private val RUNTIME_NAME = "Runtime" /** Get or create the runtime which bound with Spark TaskContext. */ - def contextInstance(): Runtime = { + def contextInstance(name: String): Runtime = { if (!TaskResources.inSparkTask()) { throw new IllegalStateException("This method must be called in a Spark task.") } - TaskResources.addResourceIfNotRegistered(RUNTIME_NAME, () => create()) + TaskResources.addResourceIfNotRegistered(name, () => create(name)) } - private def create(): Runtime = { - new Runtime + private def create(name: String): Runtime with TaskResource = { + Runtime(name) } } diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index 326f836a0a58..e75abe41e4e8 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -19,7 +19,6 @@ package org.apache.gluten.vectorized import org.apache.gluten.GlutenConfig import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.ArrowAbiUtil import org.apache.spark.SparkEnv @@ -79,7 +78,6 @@ private class ColumnarBatchSerializerInstance( extends SerializerInstance with Logging { - private val nmm = NativeMemoryManagers.contextInstance("ShuffleReader") private val shuffleReaderHandle = { val allocator: BufferAllocator = ArrowBufferAllocators .contextInstance(classOf[ColumnarBatchSerializerInstance].getSimpleName) @@ -98,15 +96,14 @@ private class ColumnarBatchSerializerInstance( val compressionCodecBackend = GlutenConfig.getConf.columnarShuffleCodecBackend.orNull val batchSize = GlutenConfig.getConf.maxBatchSize - val jniWrapper = ShuffleReaderJniWrapper.create() + val runtime = Runtimes.contextInstance("ShuffleReader") + val jniWrapper = ShuffleReaderJniWrapper.create(runtime) val shuffleReaderHandle = jniWrapper.make( cSchema.memoryAddress(), - nmm.getNativeInstanceHandle, compressionCodec, compressionCodecBackend, batchSize, - shuffleWriterType - ) + shuffleWriterType) // Close shuffle reader instance as lately as the end of task processing, // since the native reader could hold a reference to memory pool that // was used to create all buffers read from shuffle reader. The pool @@ -137,12 +134,12 @@ private class ColumnarBatchSerializerInstance( extends DeserializationStream with TaskResource { private val byteIn: JniByteInputStream = JniByteInputStreams.create(in) + private val runtime = Runtimes.contextInstance("ShuffleReader") private val wrappedOut: GeneralOutIterator = new ColumnarBatchOutIterator( - Runtimes.contextInstance(), + runtime, ShuffleReaderJniWrapper - .create() - .readStream(shuffleReaderHandle, byteIn), - nmm) + .create(runtime) + .readStream(shuffleReaderHandle, byteIn)) private var cb: ColumnarBatch = _ diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala index af004b54fd93..4e59d0ba69a9 100644 --- a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala +++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala @@ -18,10 +18,8 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.memtarget.MemoryTarget -import org.apache.gluten.memory.memtarget.Spiller -import org.apache.gluten.memory.memtarget.Spillers -import org.apache.gluten.memory.nmm.NativeMemoryManagers +import org.apache.gluten.exec.Runtimes +import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.vectorized._ import org.apache.spark._ @@ -98,7 +96,9 @@ class ColumnarShuffleWriter[K, V]( private val reallocThreshold = GlutenConfig.getConf.columnarShuffleReallocThreshold - private val jniWrapper = ShuffleWriterJniWrapper.create() + private val runtime = Runtimes.contextInstance("ShuffleWriter") + + private val jniWrapper = ShuffleWriterJniWrapper.create(runtime) private var nativeShuffleWriter: Long = -1L @@ -153,36 +153,25 @@ class ColumnarShuffleWriter[K, V]( dataTmp.getAbsolutePath, blockManager.subDirsPerLocalDir, localDirs, - NativeMemoryManagers - .create( - "ShuffleWriter", - new Spiller() { - override def spill(self: MemoryTarget, size: Long): Long = { - if (nativeShuffleWriter == -1L) { - throw new IllegalStateException( - "Fatal: spill() called before a shuffle writer " + - "is created. This behavior should be optimized by moving memory " + - "allocations from make() to split()") - } - logInfo(s"Gluten shuffle writer: Trying to spill $size bytes of data") - // fixme pass true when being called by self - val spilled = - jniWrapper.nativeEvict(nativeShuffleWriter, size, false) - logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data") - spilled - } - - override def applicablePhases(): java.util.Set[Spiller.Phase] = - Spillers.PHASE_SET_SPILL_ONLY - } - ) - .getNativeInstanceHandle, reallocThreshold, handle, taskContext.taskAttemptId(), GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, taskContext.partitionId), shuffleWriterType ) + runtime.addSpiller(new Spiller() { + override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = { + if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { + return 0L + } + logInfo(s"Gluten shuffle writer: Trying to spill $size bytes of data") + // fixme pass true when being called by self + val spilled = + jniWrapper.nativeEvict(nativeShuffleWriter, size, false) + logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data") + spilled + } + }) } val startTime = System.nanoTime() jniWrapper.write(nativeShuffleWriter, rows, handle, availableOffHeapPerTask()) diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index 840f8618b0b4..2f6abdc370d6 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.execution import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators @@ -41,7 +40,8 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra extends BuildSideRelation { override def deserialized: Iterator[ColumnarBatch] = { - val jniWrapper = ColumnarBatchSerializerJniWrapper.create() + val runtime = Runtimes.contextInstance("BuildSideRelation#deserialized") + val jniWrapper = ColumnarBatchSerializerJniWrapper.create(runtime) val serializeHandle: Long = { val allocator = ArrowBufferAllocators.contextInstance() val cSchema = ArrowSchema.allocateNew(allocator) @@ -50,11 +50,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra SQLConf.get.sessionLocalTimeZone) ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema) val handle = jniWrapper - .init( - cSchema.memoryAddress(), - NativeMemoryManagers - .contextInstance("BuildSideRelation#BatchSerializer") - .getNativeInstanceHandle) + .init(cSchema.memoryAddress()) cSchema.close() handle } @@ -72,7 +68,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra jniWrapper .deserialize(serializeHandle, batches(batchId)) batchId += 1 - ColumnarBatches.create(Runtimes.contextInstance(), handle) + ColumnarBatches.create(runtime, handle) } }) .protectInvocationFlow() @@ -90,10 +86,9 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra * was called in Spark Driver, should manage resources carefully. */ override def transform(key: Expression): Array[InternalRow] = TaskResources.runUnsafe { + val runtime = Runtimes.contextInstance("BuildSideRelation#transform") // This transformation happens in Spark driver, thus resources can not be managed automatically. - val runtime = Runtimes.contextInstance() - val nativeMemoryManager = NativeMemoryManagers.contextInstance("BuildSideRelation#transform") - val serializerJniWrapper = ColumnarBatchSerializerJniWrapper.create() + val serializerJniWrapper = ColumnarBatchSerializerJniWrapper.create(runtime) val serializeHandle = { val allocator = ArrowBufferAllocators.contextInstance() val cSchema = ArrowSchema.allocateNew(allocator) @@ -101,8 +96,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra SparkShimLoader.getSparkShims.structFromAttributes(output), SQLConf.get.sessionLocalTimeZone) ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema) - val handle = serializerJniWrapper - .init(cSchema.memoryAddress(), nativeMemoryManager.getNativeInstanceHandle) + val handle = serializerJniWrapper.init(cSchema.memoryAddress()) cSchema.close() handle } @@ -110,8 +104,8 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra var closed = false // Convert columnar to Row. - val jniWrapper = NativeColumnarToRowJniWrapper.create() - val c2rId = jniWrapper.nativeColumnarToRowInit(nativeMemoryManager.getNativeInstanceHandle) + val jniWrapper = NativeColumnarToRowJniWrapper.create(runtime) + val c2rId = jniWrapper.nativeColumnarToRowInit() var batchId = 0 val iterator = if (batches.length > 0) { val res: Iterator[Iterator[InternalRow]] = new Iterator[Iterator[InternalRow]] { @@ -142,7 +136,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra val cols = batch.numCols() val rows = batch.numRows() val info = - jniWrapper.nativeColumnarToRowConvert(batchHandle, c2rId) + jniWrapper.nativeColumnarToRowConvert(c2rId, ColumnarBatches.getNativeHandle(batch)) batch.close() val columnNames = key.flatMap { case expression: AttributeReference => diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index 090b8fa2562a..22b376a1b608 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.utils import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.memory.nmm.NativeMemoryManagers import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning} @@ -41,14 +41,12 @@ import org.apache.spark.util.MutablePair object ExecUtil { def convertColumnarToRow(batch: ColumnarBatch): Iterator[InternalRow] = { - val jniWrapper = NativeColumnarToRowJniWrapper.create() + val runtime = Runtimes.contextInstance("ExecUtil#ColumnarToRow") + val jniWrapper = NativeColumnarToRowJniWrapper.create(runtime) var info: NativeColumnarToRowInfo = null val batchHandle = ColumnarBatches.getNativeHandle(batch) - val c2rHandle = jniWrapper.nativeColumnarToRowInit( - NativeMemoryManagers - .contextInstance("ExecUtil#ColumnarToRow") - .getNativeInstanceHandle) - info = jniWrapper.nativeColumnarToRowConvert(batchHandle, c2rHandle) + val c2rHandle = jniWrapper.nativeColumnarToRowInit() + info = jniWrapper.nativeColumnarToRowConvert(c2rHandle, batchHandle) Iterators .wrap(new Iterator[InternalRow] { @@ -147,7 +145,10 @@ object ExecUtil { val newHandle = ColumnarBatches.compose(pidBatch, cb) // Composed batch already hold pidBatch's shared ref, so close is safe. ColumnarBatches.forceClose(pidBatch) - (0, ColumnarBatches.create(ColumnarBatches.getRuntime(cb), newHandle)) + ( + 0, + ColumnarBatches + .create(Runtimes.contextInstance("ExecUtil#getShuffleDependency"), newHandle)) }) .recyclePayload(p => ColumnarBatches.forceClose(p._2)) // FIXME why force close? .create() diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index 471d70f4d1a5..bd205ba7a469 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -18,10 +18,11 @@ import org.apache.gluten.GlutenConfig; import org.apache.gluten.columnarbatch.ColumnarBatches; +import org.apache.gluten.exec.Runtime; +import org.apache.gluten.exec.Runtimes; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; -import org.apache.gluten.memory.nmm.NativeMemoryManagers; import org.apache.gluten.vectorized.ShuffleWriterJniWrapper; import org.apache.gluten.vectorized.SplitResult; @@ -45,7 +46,6 @@ import java.io.IOException; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -67,7 +67,8 @@ public class VeloxUniffleColumnarShuffleWriter extends RssShuffleWriter> records) throws IOException { bufferSize, bufferSize, partitionPusher, - NativeMemoryManagers.create( - "UniffleShuffleWriter", - new Spiller() { - @Override - public long spill(MemoryTarget self, long size) { - if (nativeShuffleWriter == -1) { - throw new IllegalStateException( - "Fatal: spill() called before a shuffle shuffle writer " - + "evaluator is created. This behavior should be" - + "optimized by moving memory " - + "allocations from make() to split()"); - } - LOG.info( - "Gluten shuffle writer: Trying to push {} bytes of data", size); - long pushed = - jniWrapper.nativeEvict(nativeShuffleWriter, size, false); - LOG.info( - "Gluten shuffle writer: Pushed {} / {} bytes of data", - pushed, - size); - return pushed; - } - - @Override - public Set applicablePhases() { - return Spillers.PHASE_SET_SPILL_ONLY; - } - }) - .getNativeInstanceHandle(), handle, taskAttemptId, GlutenShuffleUtils.getStartPartitionId( @@ -186,6 +158,26 @@ public Set applicablePhases() { "uniffle", isSort ? "sort" : "hash", reallocThreshold); + runtime.addSpiller( + new Spiller() { + @Override + public long spill(MemoryTarget self, Spiller.Phase phase, long size) { + if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { + return 0L; + } + if (nativeShuffleWriter == -1) { + throw new IllegalStateException( + "Fatal: spill() called before a shuffle shuffle writer " + + "evaluator is created. This behavior should be" + + "optimized by moving memory " + + "allocations from make() to split()"); + } + LOG.info("Gluten shuffle writer: Trying to push {} bytes of data", size); + long pushed = jniWrapper.nativeEvict(nativeShuffleWriter, size, false); + LOG.info("Gluten shuffle writer: Pushed {} / {} bytes of data", pushed, size); + return pushed; + } + }); } long startTime = System.nanoTime(); long bytes = @@ -242,7 +234,7 @@ public Option stop(boolean success) { closeShuffleWriter(); return super.stop(success); } - return Option.empty(); + return Option.empty(); } private void closeShuffleWriter() { From c5ce552280976eba884bcae36bb02b8afc355017 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Mon, 1 Jul 2024 14:49:03 +0700 Subject: [PATCH 369/402] [CELEBORN][DOC] Fix Celeborn support of get-started document (#6282) --- docs/get-started/ClickHouse.md | 33 +++++++++++---------------------- docs/get-started/Velox.md | 4 ++-- docs/get-started/build-guide.md | 23 ++++++++++++----------- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md index ab24de7a4fd6..38ce048fe0de 100644 --- a/docs/get-started/ClickHouse.md +++ b/docs/get-started/ClickHouse.md @@ -629,19 +629,26 @@ public read-only account：gluten/hN2xX3uQ4m ### Celeborn support -Gluten with clickhouse backend has not yet supportted [Celeborn](https://github.com/apache/celeborn) natively as remote shuffle service using columar shuffle. However, you can still use Celeborn with row shuffle, which means a ColumarBatch will be converted to a row during shuffle. -Below introduction is used to enable this feature: +Gluten with clickhouse backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`. + +Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. +When compiling the Gluten Java module, it's required to enable `celeborn` profile, as follows: + +``` +mvn clean package -Pbackends-clickhouse -Pspark-3.3 -Pceleborn -DskipTests +``` + Then add the Spark Celeborn Client packages to your Spark application's classpath(usually add them into `$SPARK_HOME/jars`). - Celeborn: celeborn-client-spark-3-shaded_2.12-[celebornVersion].jar -Currently to use Celeborn following configurations are required in `spark-defaults.conf` +Currently to use Gluten following configurations are required in `spark-defaults.conf` ``` -spark.shuffle.manager org.apache.spark.shuffle.celeborn.SparkShuffleManager +spark.shuffle.manager org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager # celeborn master spark.celeborn.master.endpoints clb-master:9097 @@ -670,24 +677,6 @@ spark.celeborn.storage.hdfs.dir hdfs:///celeborn spark.dynamicAllocation.enabled false ``` -#### Celeborn Columnar Shuffle Support -Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`. -The native Celeborn support can be enabled by the following configuration -``` -spark.shuffle.manager=org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager -``` - -quickly start a celeborn cluster -```shell -wget https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz && \ -tar -zxvf apache-celeborn-0.3.2-incubating-bin.tgz && \ -mv apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf.template apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \ -mv apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml.template apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml && \ -mkdir /opt/hadoop && chmod 777 /opt/hadoop && \ -echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs /tmp\nceleborn.worker.monitor.disk.enabled false" > apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \ -bash apache-celeborn-0.3.2-incubating-bin/sbin/start-master.sh && bash apache-celeborn-0.3.2-incubating-bin/sbin/start-worker.sh -``` - ### Columnar shuffle mode We have two modes of columnar shuffle 1. prefer cache diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index d65b94fc1c26..5f9ae2a46b19 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -224,11 +224,11 @@ Currently there are several ways to asscess S3 in Spark. Please refer [Velox S3] Gluten with velox backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`. -Below introduction is used to enable this feature +Below introduction is used to enable this feature. First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster. -When compiling the Gluten Java module, it's required to enable `rss` profile, as follows: +When compiling the Gluten Java module, it's required to enable `celeborn` profile, as follows: ``` mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index b2e4b9560301..dc4989bc8642 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -55,17 +55,18 @@ Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`. ### Maven build parameters The below parameters can be set via `-P` for mvn. -| Parameters | Description | Default state | -|---------------------|------------------------------------------------------------------------------|---------------| -| backends-velox | Build Gluten Velox backend. | disabled | -| backends-clickhouse | Build Gluten ClickHouse backend. | disabled | -| rss | Build Gluten with Remote Shuffle Service, only applicable for Velox backend. | disabled | -| delta | Build Gluten with Delta Lake support. | disabled | -| iceberg | Build Gluten with Iceberg support. | disabled | -| spark-3.2 | Build Gluten for Spark 3.2. | enabled | -| spark-3.3 | Build Gluten for Spark 3.3. | disabled | -| spark-3.4 | Build Gluten for Spark 3.4. | disabled | -| spark-3.5 | Build Gluten for Spark 3.5. | disabled | +| Parameters | Description | Default state | +|---------------------|---------------------------------------|---------------| +| backends-velox | Build Gluten Velox backend. | disabled | +| backends-clickhouse | Build Gluten ClickHouse backend. | disabled | +| celeborn | Build Gluten with Celeborn. | disabled | +| uniffle | Build Gluten with Uniffle. | disabled | +| delta | Build Gluten with Delta Lake support. | disabled | +| iceberg | Build Gluten with Iceberg support. | disabled | +| spark-3.2 | Build Gluten for Spark 3.2. | enabled | +| spark-3.3 | Build Gluten for Spark 3.3. | disabled | +| spark-3.4 | Build Gluten for Spark 3.4. | disabled | +| spark-3.5 | Build Gluten for Spark 3.5. | disabled | ## Gluten Jar for Deployment The gluten jar built out is under `GLUTEN_SRC/package/target/`. From dc3e22bd7be73fff9c77fd492204f7ca67198de0 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Mon, 1 Jul 2024 20:53:27 +0800 Subject: [PATCH 370/402] [CI] Fix centos7 CI build error (#6298) --- .github/workflows/velox_docker.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 098b2a2d57eb..ded2032f4241 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -192,10 +192,11 @@ jobs: name: velox-arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list - if: matrix.os == 'centos:8' run: | - sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true - sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + if [ "${{ matrix.os }}" = "centos:7" ] || [ "${{ matrix.os }}" = "centos:8" ]; then + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + fi - name: Setup java and maven run: | if [ "${{ matrix.java }}" = "java-17" ]; then From 5d6d214f00f0ce2bdb67ac786d5be244026427c6 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Tue, 2 Jul 2024 00:12:37 +0800 Subject: [PATCH 371/402] [VL] Daily Update Velox Version (2024_06_30) (#6284) 0ef0ac8e4 by Jia Ke, Enable right join in smj (10148) c54e59dbb by wypb, Fix HashStringAllocator::clear() and cumulativeBytes_ (10260) 4963d7116 by duanmeng, Add recursive spill for RowNumber (8654) e3de4ea9d by Sandino Flores, Add support for Protobuf v22+ (10294) 0d8022846 by PHILO-HE, Support finding installed arrow libraries from system (9992) fd955bff4 by liangyongyuan, Add float/double types support for Spark mod function (9848) 0ced9e5f0 by NEUpanning, Fix typo in expression evaluation documentation (10304) 8803bfbd1 by lingbin, Fix typo in SIMD document (10319) bcfc8f8c3 by PHILO-HE, Allow returning Status from callNullable and callNullFree methods (10274) 258db516d by PHILO-HE, Use legacySizeOfNull argument to determine the behavior of Spark size function (10100) --- .../gluten/execution/TestOperator.scala | 8 +-- cpp/CMakeLists.txt | 8 ++- cpp/core/config/GlutenConfig.h | 2 - cpp/velox/compute/WholeStageResultIterator.cc | 2 - ep/build-velox/src/build_velox.sh | 2 + ep/build-velox/src/get_velox.sh | 10 ++-- ep/build-velox/src/modify_velox.patch | 52 +++++++++---------- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 9b47a519cd28..d84f5e7cc318 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1017,7 +1017,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - test("test explode/posexplode function") { + ignore("test explode/posexplode function") { Seq("explode", "posexplode").foreach { func => // Literal: func(literal) @@ -1190,7 +1190,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla |""".stripMargin)(_) } - test("test multi-generate") { + ignore("test multi-generate") { withTable("t") { sql("CREATE TABLE t (col1 array>, col2 array) using parquet") sql("INSERT INTO t VALUES (array(struct(1, 'a'), struct(2, 'b')), array(1, 2))") @@ -1588,7 +1588,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - test("test array literal") { + ignore("test array literal") { withTable("array_table") { sql("create table array_table(a array) using parquet") sql("insert into table array_table select array(1)") @@ -1601,7 +1601,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - test("test map literal") { + ignore("test map literal") { withTable("map_table") { sql("create table map_table(a map) using parquet") sql("insert into table map_table select map(1, 'hello')") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3ee336dd6a14..c5cbab0697bf 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -68,9 +68,13 @@ if(NOT DEFINED VELOX_HOME) endif() if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") - set(ARROW_HOME ${VELOX_HOME}/_build/debug/third_party/arrow_ep) + set(ARROW_HOME + ${VELOX_HOME}/_build/debug/CMake/resolve_dependency_modules/arrow/arrow_ep/ + ) else() - set(ARROW_HOME ${VELOX_HOME}/_build/release/third_party/arrow_ep) + set(ARROW_HOME + ${VELOX_HOME}/_build/release/CMake/resolve_dependency_modules/arrow/arrow_ep + ) endif() include(ResolveDependency) diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index a039537b78ba..ad7dacf113ec 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -30,8 +30,6 @@ const std::string kGlutenSaveDir = "spark.gluten.saveDir"; const std::string kCaseSensitive = "spark.sql.caseSensitive"; -const std::string kLegacySize = "spark.sql.legacy.sizeOfNull"; - const std::string kSessionTimezone = "spark.sql.session.timeZone"; const std::string kIgnoreMissingFiles = "spark.sql.files.ignoreMissingFiles"; diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index cbc6c838b1b7..296b9415b159 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -454,8 +454,6 @@ std::unordered_map WholeStageResultIterator::getQueryC } // Adjust timestamp according to the above configured session timezone. configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true"; - // Align Velox size function with Spark. - configs[velox::core::QueryConfig::kSparkLegacySizeOfNull] = std::to_string(veloxCfg_->get(kLegacySize, true)); { // partial aggregation memory config diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index b812b6b52bd6..b55f65a98e9e 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -147,6 +147,8 @@ function compile { echo "NUM_THREADS_OPTS: $NUM_THREADS_OPTS" export simdjson_SOURCE=AUTO + # Quick fix for CI error due to velox rebase + export Arrow_SOURCE=BUNDLED if [ $ARCH == 'x86_64' ]; then make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' ]]; then diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 0adc1ce8ff61..808e48881ea7 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_28 +VELOX_BRANCH=2024_06_30 VELOX_HOME="" #Set on run gluten on HDFS @@ -256,11 +256,11 @@ function apply_compilation_fixes { current_dir=$1 velox_home=$2 sudo cp ${current_dir}/modify_velox.patch ${velox_home}/ - sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/third_party/ - sudo cp ${current_dir}/modify_arrow_dataset_scan_option.patch ${velox_home}/third_party/ + sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/CMake/resolve_dependency_modules/arrow/ + sudo cp ${current_dir}/modify_arrow_dataset_scan_option.patch ${velox_home}/CMake/resolve_dependency_modules/arrow/ git add ${velox_home}/modify_velox.patch # to avoid the file from being deleted by git clean -dffx :/ - git add ${velox_home}/third_party/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/ - git add ${velox_home}/third_party/modify_arrow_dataset_scan_option.patch # to avoid the file from being deleted by git clean -dffx :/ + git add ${velox_home}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/ + git add ${velox_home}/CMake/resolve_dependency_modules/arrow/modify_arrow_dataset_scan_option.patch # to avoid the file from being deleted by git clean -dffx :/ cd ${velox_home} echo "Applying patch to Velox source code..." git apply modify_velox.patch diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch index aee406c3eae0..cc05d3f91f9c 100644 --- a/ep/build-velox/src/modify_velox.patch +++ b/ep/build-velox/src/modify_velox.patch @@ -35,8 +35,31 @@ index d49115f12..1aaa8e532 100644 + IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}") + endif() endif() +diff --git a/CMake/resolve_dependency_modules/arrow/CMakeLists.txt b/CMake/resolve_dependency_modules/arrow/CMakeLists.txt +index 3f01df2fd..8c1c493f3 100644 +--- a/CMake/resolve_dependency_modules/arrow/CMakeLists.txt ++++ b/CMake/resolve_dependency_modules/arrow/CMakeLists.txt +@@ -24,6 +24,9 @@ if(VELOX_ENABLE_ARROW) + set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep") + set(ARROW_CMAKE_ARGS + -DARROW_PARQUET=OFF ++ -DARROW_PARQUET=ON ++ -DARROW_FILESYSTEM=ON ++ -DARROW_PROTOBUF_USE_SHARED=OFF + -DARROW_WITH_THRIFT=ON + -DARROW_WITH_LZ4=ON + -DARROW_WITH_SNAPPY=ON +@@ -66,6 +69,8 @@ if(VELOX_ENABLE_ARROW) + arrow_ep + PREFIX ${ARROW_PREFIX} + URL ${VELOX_ARROW_SOURCE_URL} ++ PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow.patch ++ COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch + URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM} + SOURCE_SUBDIR cpp + CMAKE_ARGS ${ARROW_CMAKE_ARGS} diff --git a/CMakeLists.txt b/CMakeLists.txt -index 5c7bf770a..9f897f577 100644 +index bb7c49907..3372d48b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,10 +234,15 @@ if(VELOX_ENABLE_ABFS) @@ -59,7 +82,7 @@ index 5c7bf770a..9f897f577 100644 add_definitions(-DVELOX_ENABLE_HDFS3) endif() -@@ -377,7 +382,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES}) +@@ -378,7 +383,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES}) # for reference. find_package(range-v3) set_source(gflags) @@ -68,31 +91,6 @@ index 5c7bf770a..9f897f577 100644 if(NOT TARGET gflags::gflags) # This is a bit convoluted, but we want to be able to use gflags::gflags as a # target even when velox is built as a subproject which uses - -diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt -index ce4c24dbe..785a2acc6 100644 ---- a/third_party/CMakeLists.txt -+++ b/third_party/CMakeLists.txt -@@ -26,7 +26,9 @@ if(VELOX_ENABLE_ARROW) - endif() - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep") - set(ARROW_CMAKE_ARGS -- -DARROW_PARQUET=OFF -+ -DARROW_PARQUET=ON -+ -DARROW_FILESYSTEM=ON -+ -DARROW_PROTOBUF_USE_SHARED=OFF - -DARROW_WITH_THRIFT=ON - -DARROW_WITH_LZ4=ON - -DARROW_WITH_SNAPPY=ON -@@ -69,6 +71,8 @@ if(VELOX_ENABLE_ARROW) - arrow_ep - PREFIX ${ARROW_PREFIX} - URL ${VELOX_ARROW_SOURCE_URL} -+ PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow.patch -+ COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch - URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM} - SOURCE_SUBDIR cpp - CMAKE_ARGS ${ARROW_CMAKE_ARGS} diff --git a/velox/common/process/tests/CMakeLists.txt b/velox/common/process/tests/CMakeLists.txt index 6797697a1..3e241f8f7 100644 --- a/velox/common/process/tests/CMakeLists.txt From 15c99ba093440766d332f85fd873961c47774f66 Mon Sep 17 00:00:00 2001 From: lgbo Date: Tue, 2 Jul 2024 09:56:02 +0800 Subject: [PATCH 372/402] ensure same results from NaN (#6301) --- cpp-ch/local-engine/Functions/SparkFunctionHashingExtended.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp-ch/local-engine/Functions/SparkFunctionHashingExtended.h b/cpp-ch/local-engine/Functions/SparkFunctionHashingExtended.h index 82d678aa2a4e..57bf00ba9904 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionHashingExtended.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionHashingExtended.h @@ -28,6 +28,7 @@ #include #include #include +#include namespace DB { @@ -200,14 +201,14 @@ class SparkFunctionAnyHash : public IFunction { if constexpr (std::is_same_v) { - if (n == -0.0f) [[unlikely]] + if (n == -0.0f || isNaN(n)) [[unlikely]] return applyNumber(0, seed); else return Impl::apply(reinterpret_cast(&n), sizeof(n), seed); } else { - if (n == -0.0) [[unlikely]] + if (n == -0.0 || isNaN(n)) [[unlikely]] return applyNumber(0, seed); else return Impl::apply(reinterpret_cast(&n), sizeof(n), seed); From 7a3c129c5eb1359dfa4d8346373ca0eca89c6db8 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 2 Jul 2024 10:31:31 +0800 Subject: [PATCH 373/402] [VL] Disable protobuf build by default (#6297) --- dev/builddeps-veloxbe.sh | 4 ++-- docs/get-started/build-guide.md | 4 ++-- ep/build-velox/src/get_velox.sh | 23 +++++------------------ 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index d5e33e926d82..4e0882a834dc 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -13,7 +13,7 @@ BUILD_TESTS=OFF BUILD_EXAMPLES=OFF BUILD_BENCHMARKS=OFF BUILD_JEMALLOC=OFF -BUILD_PROTOBUF=ON +BUILD_PROTOBUF=OFF BUILD_VELOX_TESTS=OFF BUILD_VELOX_BENCHMARKS=OFF ENABLE_QAT=OFF @@ -201,7 +201,7 @@ function build_arrow { function build_velox { echo "Start to build Velox" cd $GLUTEN_DIR/ep/build-velox/src - ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --build_protobuf=$BUILD_PROTOBUF --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER + ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER # When BUILD_TESTS is on for gluten cpp, we need turn on VELOX_BUILD_TEST_UTILS via build_test_utils. ./build_velox.sh --run_setup_script=$RUN_SETUP_SCRIPT --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ --enable_abfs=$ENABLE_ABFS --enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS \ diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index dc4989bc8642..a9e9bd1266df 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -14,8 +14,8 @@ Please set them via `--`, e.g. `--build_type=Release`. | build_tests | Build gluten cpp tests. | OFF | | build_examples | Build udf example. | OFF | | build_benchmarks | Build gluten cpp benchmarks. | OFF | -| build_jemalloc | Build with jemalloc. | OFF | -| build_protobuf | Build protobuf lib. | ON | +| build_jemalloc | Build with jemalloc. | OFF | +| build_protobuf | Build protobuf lib. | OFF | | enable_qat | Enable QAT for shuffle data de/compression. | OFF | | enable_iaa | Enable IAA for shuffle data de/compression. | OFF | | enable_hbm | Enable HBM allocator. | OFF | diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 808e48881ea7..bbc147e55f81 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -22,8 +22,6 @@ VELOX_HOME="" #Set on run gluten on HDFS ENABLE_HDFS=OFF -#It can be set to OFF when compiling velox again -BUILD_PROTOBUF=ON #Set on run gluten on S3 ENABLE_S3=OFF #Set on run gluten on GCS @@ -47,10 +45,6 @@ for arg in "$@"; do VELOX_HOME=("${arg#*=}") shift # Remove argument name from processing ;; - --build_protobuf=*) - BUILD_PROTOBUF=("${arg#*=}") - shift # Remove argument name from processing - ;; --enable_hdfs=*) ENABLE_HDFS=("${arg#*=}") shift # Remove argument name from processing @@ -95,15 +89,13 @@ function process_setup_ubuntu { sed -i '/ccache/a\ curl \\' scripts/setup-ubuntu.sh sed -i '/libgmock-dev/d' scripts/setup-ubuntu.sh # resolved by ep/build-velox/build/velox_ep/CMake/resolve_dependency_modules/gtest.cmake sed -i 's/github_checkout boostorg\/boost \"\${BOOST_VERSION}\" --recursive/wget_and_untar https:\/\/github.com\/boostorg\/boost\/releases\/download\/boost-1.84.0\/boost-1.84.0.tar.gz boost \&\& cd boost/g' scripts/setup-ubuntu.sh + sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh + sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh if [ $ENABLE_HDFS == "ON" ]; then sed -i '/^function install_folly.*/i function install_libhdfs3 {\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-ubuntu.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh fi - if [ $BUILD_PROTOBUF == "ON" ]; then - sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh - fi sed -i "s/apt install -y/sudo apt install -y/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh @@ -136,15 +128,14 @@ function process_setup_centos8 { sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos8.sh sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos8.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos8.sh + sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh if [ $ENABLE_HDFS == "ON" ]; then sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos8.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos8.sh sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos8.sh fi - if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then - sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh - fi + sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos8.sh @@ -172,15 +163,12 @@ function process_setup_centos7 { # install gtest sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_gtest' scripts/setup-centos7.sh - + sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh if [ $ENABLE_HDFS = "ON" ]; then sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-centos7.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh fi - if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh - fi sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos7.sh @@ -219,7 +207,6 @@ function process_setup_tencentos32 { echo "Preparing Velox source code..." echo "ENABLE_HDFS=${ENABLE_HDFS}" -echo "BUILD_PROTOBUF=${BUILD_PROTOBUF}" CURRENT_DIR=$( cd "$(dirname "$BASH_SOURCE")" From 456316d28e7f7740184cc39d093eebf029300110 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:42:07 +0800 Subject: [PATCH 374/402] [VL] Daily Update Velox Version (2024_07_02) (#6303) 6caa7d535 by duanmeng, Add max spill level in RowNumber fuzzer (10347) 181c6aec1 by willsfeng, Add secure_random() and secure_random(lower,upper) Presto functions (9295) 4f532a033 by Deepak Majeti, Fix failing Linux release with adapters (10360) 02dca9d39 by aditi-pandit, Fix spellings in MemoryAllocator code (10350) a442baac5 by Masha Basmanova, Optimize cast(uuid as varchar) 296568087 by Daniel Hunte, Clean up yearTimestampWithTimezone test (10344) --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index bbc147e55f81..54ba3e070e28 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_06_30 +VELOX_BRANCH=2024_07_02 VELOX_HOME="" #Set on run gluten on HDFS From 83a3c0f79854f2cc855482da4a48ed9dcf333809 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 2 Jul 2024 12:55:28 +0800 Subject: [PATCH 375/402] [GLUTEN-5248][VL] Directly pass legacySizeOfNull to native size function (#6014) --- .../backendsapi/clickhouse/CHSparkPlanExecApi.scala | 7 ------- .../gluten/expression/CHExpressionTransformer.scala | 10 ---------- .../execution/ScalarFunctionsValidateSuite.scala | 6 ++++++ .../org/apache/gluten/execution/TestOperator.scala | 8 ++++---- .../apache/gluten/backendsapi/SparkPlanExecApi.scala | 7 ------- .../gluten/expression/ExpressionConverter.scala | 12 +++++------- 6 files changed, 15 insertions(+), 35 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index ac3ea61ff810..7ed333aec254 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -612,13 +612,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { CHStringTranslateTransformer(substraitExprName, srcExpr, matchingExpr, replaceExpr, original) } - override def genSizeExpressionTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Size): ExpressionTransformer = { - CHSizeExpressionTransformer(substraitExprName, child, original) - } - override def genLikeTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala index 5ca4e02339d0..7b389ead0091 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala @@ -30,16 +30,6 @@ import com.google.common.collect.Lists import java.util.Locale -case class CHSizeExpressionTransformer( - substraitExprName: String, - expr: ExpressionTransformer, - original: Size) - extends BinaryExpressionTransformer { - override def left: ExpressionTransformer = expr - // Pass legacyLiteral as second argument in substrait function - override def right: ExpressionTransformer = LiteralTransformer(original.legacySizeOfNull) -} - case class CHTruncTimestampTransformer( substraitExprName: String, format: ExpressionTransformer, diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index bd32a799c3ac..3db0f5e79b75 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -101,6 +101,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) { + runQueryAndCompare("SELECT array_size(null)") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("chr function") { val df = runQueryAndCompare( "SELECT chr(l_orderkey + 64) " + diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index d84f5e7cc318..9b47a519cd28 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1017,7 +1017,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - ignore("test explode/posexplode function") { + test("test explode/posexplode function") { Seq("explode", "posexplode").foreach { func => // Literal: func(literal) @@ -1190,7 +1190,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla |""".stripMargin)(_) } - ignore("test multi-generate") { + test("test multi-generate") { withTable("t") { sql("CREATE TABLE t (col1 array>, col2 array) using parquet") sql("INSERT INTO t VALUES (array(struct(1, 'a'), struct(2, 'b')), array(1, 2))") @@ -1588,7 +1588,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - ignore("test array literal") { + test("test array literal") { withTable("array_table") { sql("create table array_table(a array) using parquet") sql("insert into table array_table select array(1)") @@ -1601,7 +1601,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - ignore("test map literal") { + test("test map literal") { withTable("map_table") { sql("create table map_table(a map) using parquet") sql("insert into table map_table select map(1, 'hello')") diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 3ca5e0313924..ff7449e2d340 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -464,13 +464,6 @@ trait SparkPlanExecApi { original) } - def genSizeExpressionTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Size): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(child), original) - } - def genLikeTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index d5222cfc6350..b5bcb6876e4d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -396,14 +396,12 @@ object ExpressionConverter extends SQLConfHelper with Logging { r ) case size: Size => - if (size.legacySizeOfNull != SQLConf.get.legacySizeOfNull) { - throw new GlutenNotSupportException( - "The value of legacySizeOfNull field of size is " + - "not equals to legacySizeOfNull of SQLConf, this case is not supported yet") - } - BackendsApiManager.getSparkPlanExecApiInstance.genSizeExpressionTransformer( + // Covers Spark ArraySize which is replaced by Size(child, false). + val child = + replaceWithExpressionTransformerInternal(size.child, attributeSeq, expressionsMap) + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal(size.child, attributeSeq, expressionsMap), + Seq(child, LiteralTransformer(size.legacySizeOfNull)), size) case namedStruct: CreateNamedStruct => BackendsApiManager.getSparkPlanExecApiInstance.genNamedStructTransformer( From 5c7c84ba37d6269c339e20c221e971f332fe7333 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 2 Jul 2024 13:22:32 +0800 Subject: [PATCH 376/402] [VL] CI: Fix CPP tests are not running (#6295) * fixup * fixup * fixup --- .github/workflows/velox_docker.yml | 8 ++++---- cpp/velox/tests/BufferOutputStreamTest.cc | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index ded2032f4241..36e81cc3ab2f 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -617,6 +617,10 @@ jobs: install_arrow_deps ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON --build_protobuf=ON + - name: Gluten CPP Test + run: | + cd ./cpp/build && \ + ctest -V - uses: actions/upload-artifact@v2 with: name: velox-native-lib-centos-8-${{github.sha}} @@ -682,10 +686,6 @@ jobs: working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - name: Gluten CPP Test - run: | - cd $GITHUB_WORKSPACE/cpp/build && \ - ctest -V - name: Prepare spark.test.home for Spark 3.2.2 (other tests) run: | cd $GITHUB_WORKSPACE/ && \ diff --git a/cpp/velox/tests/BufferOutputStreamTest.cc b/cpp/velox/tests/BufferOutputStreamTest.cc index 324d8c5e6394..0e16f8c87769 100644 --- a/cpp/velox/tests/BufferOutputStreamTest.cc +++ b/cpp/velox/tests/BufferOutputStreamTest.cc @@ -16,6 +16,7 @@ */ #include "memory/BufferOutputStream.h" +#include "compute/VeloxBackend.h" #include "memory/VeloxColumnarBatch.h" #include "velox/common/memory/ByteStream.h" #include "velox/vector/tests/utils/VectorTestBase.h" @@ -27,6 +28,7 @@ class BufferOutputStreamTest : public ::testing::Test, public test::VectorTestBa protected: // Velox requires the mem manager to be instanced. static void SetUpTestCase() { + VeloxBackend::create({}); memory::MemoryManager::testingSetInstance({}); } From eb1b913ccad5c48f067813f25639dbc004b41ffb Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 2 Jul 2024 13:26:30 +0800 Subject: [PATCH 377/402] [VL] CI: Update job `run-tpc-test-ubuntu-oom` for latest memory usage status (#6291) * fixup * debug * Revert "debug" This reverts commit a220b8c4c5d84d3f1fc2527b8bf4739c4e2401b8. --- .github/workflows/velox_docker.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 36e81cc3ab2f..d07ceb93b3d9 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -335,8 +335,7 @@ jobs: -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 - - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory, memory isolation on - if: false # Disabled as error https://gist.github.com/zhztheplayer/abd5e83ccdc48730678ae7ebae479fcc + - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory, memory isolation on # Disabled as error https://gist.github.com/zhztheplayer/abd5e83ccdc48730678ae7ebae479fcc run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ @@ -346,8 +345,8 @@ jobs: -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 - - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q97 low memory # The case currently causes crash with "free: invalid size". + -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 || true + - name: TPC-DS SF30.0 Parquet local spark3.2 Q97 low memory run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \ From 832b91c2fb3546efb45a8da3d953c35be01fa3d5 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 2 Jul 2024 14:11:30 +0800 Subject: [PATCH 378/402] [VL] Link lib gluten to arrow's static libraries (#6231) --- .../backendsapi/velox/VeloxListenerApi.scala | 20 +-------- cpp/CMake/ConfigArrow.cmake | 41 ++++++++----------- cpp/core/CMakeLists.txt | 4 +- cpp/velox/CMakeLists.txt | 5 --- dev/build_arrow.sh | 17 ++++---- dev/builddeps-veloxbe.sh | 5 +-- ep/build-velox/src/modify_velox.patch | 23 ++++++++--- 7 files changed, 48 insertions(+), 67 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 81f06478cbb6..e1abbdd7c6b7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -134,28 +134,10 @@ class VeloxListenerApi extends ListenerApi { ) { loadLibFromJar(loader, conf) } - loader - .newTransaction() - .loadAndCreateLink(s"libarrow.so.$ARROW_VERSION.0.0", s"libarrow.so.$ARROW_VERSION", false) - .loadAndCreateLink( - s"libparquet.so.$ARROW_VERSION.0.0", - s"libparquet.so.$ARROW_VERSION", - false) - .commit() } private def loadLibWithMacOS(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink( - s"libarrow.$ARROW_VERSION.0.0.dylib", - s"libarrow.$ARROW_VERSION.dylib", - false) - .loadAndCreateLink( - s"libparquet.$ARROW_VERSION.0.0.dylib", - s"libparquet.$ARROW_VERSION.dylib", - false) - .commit() + // Placeholder for loading shared libs on MacOS if user needs. } private def initialize(conf: SparkConf, isDriver: Boolean): Unit = { diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index 110836347cac..e27a3414dd31 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -15,24 +15,22 @@ # specific language governing permissions and limitations # under the License. -if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") - set(ARROW_SHARED_LIBRARY_SUFFIX ".1500.dylib") - set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".1500.1.0.dylib") -else() - set(ARROW_SHARED_LIBRARY_SUFFIX ".so.1500") - set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.1500.1.0") -endif() +set(ARROW_STATIC_LIBRARY_SUFFIX ".a") set(ARROW_LIB_NAME "arrow") set(PARQUET_LIB_NAME "parquet") -set(ARROW_DATASET_LIB_NAME "arrow_dataset") -set(ARROW_SUBSTRAIT_LIB_NAME "arrow_substrait") +set(ARROW_BUNDLED_DEPS "arrow_bundled_dependencies") + +set(ARROW_INSTALL_DIR "${ARROW_HOME}/install") +set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib") +set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64") +set(ARROW_INCLUDE_DIR "${ARROW_INSTALL_DIR}/include") function(FIND_ARROW_LIB LIB_NAME) if(NOT TARGET Arrow::${LIB_NAME}) set(ARROW_LIB_FULL_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${ARROW_SHARED_LIBRARY_SUFFIX}) - add_library(Arrow::${LIB_NAME} SHARED IMPORTED) + ${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${ARROW_STATIC_LIBRARY_SUFFIX}) + add_library(Arrow::${LIB_NAME} STATIC IMPORTED) # Firstly find the lib from velox's arrow build path. If not found, try to # find it from system. find_library( @@ -41,22 +39,17 @@ function(FIND_ARROW_LIB LIB_NAME) PATHS ${ARROW_LIB_DIR} ${ARROW_LIB64_DIR}) if(NOT ARROW_LIB_${LIB_NAME}) message(FATAL_ERROR "Arrow library Not Found: ${ARROW_LIB_FULL_NAME}") + endif() + message(STATUS "Found Arrow library: ${ARROW_LIB_${LIB_NAME}}") + if(LIB_NAME STREQUAL ${ARROW_BUNDLED_DEPS}) + set_target_properties( + Arrow::${LIB_NAME} PROPERTIES IMPORTED_LOCATION + ${ARROW_LIB_${LIB_NAME}}) else() - message(STATUS "Found Arrow library: ${ARROW_LIB_${LIB_NAME}}") set_target_properties( Arrow::${LIB_NAME} - PROPERTIES IMPORTED_LOCATION "${ARROW_LIB_${LIB_NAME}}" - INTERFACE_INCLUDE_DIRECTORIES - "${ARROW_HOME}/install/include") + PROPERTIES IMPORTED_LOCATION ${ARROW_LIB_${LIB_NAME}} + INTERFACE_INCLUDE_DIRECTORIES ${ARROW_HOME}/install/include) endif() - file( - COPY ${ARROW_LIB_${LIB_NAME}} - DESTINATION ${root_directory}/releases/ - FOLLOW_SYMLINK_CHAIN) endif() endfunction() - -set(ARROW_INSTALL_DIR "${ARROW_HOME}/install") -set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib") -set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64") -set(ARROW_INCLUDE_DIR "${ARROW_INSTALL_DIR}/include") diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index e17d13581105..cc5b6c7e926d 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -238,6 +238,7 @@ endif() find_arrow_lib(${ARROW_LIB_NAME}) find_arrow_lib(${PARQUET_LIB_NAME}) +find_arrow_lib(${ARROW_BUNDLED_DEPS}) if(ENABLE_HBM) include(BuildMemkind) @@ -314,7 +315,8 @@ else() set(LIBHDFS3_DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() -target_link_libraries(gluten PUBLIC Arrow::arrow Arrow::parquet) +target_link_libraries(gluten PUBLIC Arrow::parquet Arrow::arrow + Arrow::arrow_bundled_dependencies) target_link_libraries(gluten PRIVATE google::glog) install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 716a5f68a91c..b734669b8e46 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -590,11 +590,6 @@ endif() target_link_libraries(velox PUBLIC gluten) add_velox_dependencies() -# Arrow libraries appear after Velox dependencies to avoid linker error -find_arrow_lib(${ARROW_LIB_NAME}) -find_arrow_lib(${PARQUET_LIB_NAME}) -target_link_libraries(velox PUBLIC Arrow::arrow Arrow::parquet) - target_link_libraries(velox PUBLIC Folly::folly) find_re2() target_link_libraries(velox PUBLIC ${RE2_LIBRARY}) diff --git a/dev/build_arrow.sh b/dev/build_arrow.sh index a822c4119ea0..897dfcd267a1 100755 --- a/dev/build_arrow.sh +++ b/dev/build_arrow.sh @@ -17,15 +17,13 @@ CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) source ${CURRENT_DIR}/build_helper_functions.sh VELOX_ARROW_BUILD_VERSION=15.0.0 -ARROW_PREFIX=$CURRENT_DIR/arrow_ep -# Always uses BUNDLED in case of that thrift is not installed. -THRIFT_SOURCE="BUNDLED" +ARROW_PREFIX=$CURRENT_DIR/../ep/_ep/arrow_ep BUILD_TYPE=Release function prepare_arrow_build() { - sudo rm -rf arrow_ep/ + mkdir -p ${ARROW_PREFIX}/../ && cd ${ARROW_PREFIX}/../ && sudo rm -rf arrow_ep/ wget_and_untar https://archive.apache.org/dist/arrow/arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz arrow_ep - cd arrow_ep/ + cd arrow_ep patch -p1 < $CURRENT_DIR/../ep/build-velox/src/modify_arrow.patch patch -p1 < $CURRENT_DIR/../ep/build-velox/src/modify_arrow_dataset_scan_option.patch } @@ -38,15 +36,14 @@ function install_arrow_deps { } function build_arrow_cpp() { - if [ -n "$1" ]; then - BUILD_TYPE=$1 - fi pushd $ARROW_PREFIX/cpp cmake_install \ -DARROW_PARQUET=ON \ -DARROW_FILESYSTEM=ON \ -DARROW_PROTOBUF_USE_SHARED=OFF \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_WITH_THRIFT=ON \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ @@ -59,8 +56,8 @@ function build_arrow_cpp() { -DARROW_TESTING=ON \ -DCMAKE_INSTALL_PREFIX=/usr/local \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DARROW_BUILD_STATIC=ON \ - -DThrift_SOURCE=${THRIFT_SOURCE} + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_STATIC=ON popd } diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 4e0882a834dc..6668d0871ad1 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -192,7 +192,7 @@ function build_arrow { cd $GLUTEN_DIR/dev source build_arrow.sh prepare_arrow_build - build_arrow_cpp $BUILD_TYPE + build_arrow_cpp echo "Finished building arrow CPP" build_arrow_java echo "Finished building arrow Java" @@ -208,9 +208,8 @@ function build_velox { --num_threads=$NUM_THREADS } -## compile gluten cpp function build_gluten_cpp { - echo "Start to Gluten CPP" + echo "Start to build Gluten CPP" cd $GLUTEN_DIR/cpp rm -rf build mkdir build diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch index cc05d3f91f9c..7e1f19b3c18d 100644 --- a/ep/build-velox/src/modify_velox.patch +++ b/ep/build-velox/src/modify_velox.patch @@ -36,25 +36,38 @@ index d49115f12..1aaa8e532 100644 + endif() endif() diff --git a/CMake/resolve_dependency_modules/arrow/CMakeLists.txt b/CMake/resolve_dependency_modules/arrow/CMakeLists.txt -index 3f01df2fd..8c1c493f3 100644 +index 3f01df2fd..a8da374a2 100644 --- a/CMake/resolve_dependency_modules/arrow/CMakeLists.txt +++ b/CMake/resolve_dependency_modules/arrow/CMakeLists.txt -@@ -24,6 +24,9 @@ if(VELOX_ENABLE_ARROW) +@@ -23,7 +23,11 @@ if(VELOX_ENABLE_ARROW) + set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep") set(ARROW_CMAKE_ARGS - -DARROW_PARQUET=OFF +- -DARROW_PARQUET=OFF + -DARROW_PARQUET=ON + -DARROW_FILESYSTEM=ON + -DARROW_PROTOBUF_USE_SHARED=OFF ++ -DARROW_DEPENDENCY_USE_SHARED=OFF ++ -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_WITH_THRIFT=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -@@ -66,6 +69,8 @@ if(VELOX_ENABLE_ARROW) +@@ -37,7 +41,7 @@ if(VELOX_ENABLE_ARROW) + -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}/install + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DARROW_BUILD_STATIC=ON +- -DThrift_SOURCE=${THRIFT_SOURCE}) ++ -DARROW_BUILD_SHARED=OFF) + set(ARROW_LIBDIR ${ARROW_PREFIX}/install/${CMAKE_INSTALL_LIBDIR}) + + add_library(thrift STATIC IMPORTED GLOBAL) +@@ -66,6 +70,9 @@ if(VELOX_ENABLE_ARROW) arrow_ep PREFIX ${ARROW_PREFIX} URL ${VELOX_ARROW_SOURCE_URL} + PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow.patch -+ COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch ++ COMMAND patch -p1 < ++ ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM} SOURCE_SUBDIR cpp CMAKE_ARGS ${ARROW_CMAKE_ARGS} From b0d836bac9f9a5e4bba2dbd3de861d2abac80d8d Mon Sep 17 00:00:00 2001 From: lgbo Date: Tue, 2 Jul 2024 14:31:16 +0800 Subject: [PATCH 379/402] [GLUTEN-6159][CH] Support array functions with lambda functions (#6248) What changes were proposed in this pull request? (Please fill in changes proposed in this fix) Fixes: #6159 support following array functions filter transform aggregate How was this patch tested? (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) unit tests (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) --- .../clickhouse/CHSparkPlanExecApi.scala | 18 ++ .../GlutenFunctionValidateSuite.scala | 17 ++ .../Parser/SerializedPlanParser.cpp | 13 +- .../Parser/SerializedPlanParser.h | 4 +- .../arrayHighOrderFunctions.cpp | 154 ++++++++++++ .../scalar_function_parser/lambdaFunction.cpp | 222 ++++++++++++++++++ .../scalar_function_parser/lambdaFunction.h | 23 ++ 7 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp create mode 100644 cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.h diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 7ed333aec254..c0dee707ef4f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -842,6 +842,24 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { CHGenerateExecTransformer(generator, requiredChildOutput, outer, generatorOutput, child) } + /** Transform array filter to Substrait. */ + override def genArrayFilterTransformer( + substraitExprName: String, + argument: ExpressionTransformer, + function: ExpressionTransformer, + expr: ArrayFilter): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(argument, function), expr) + } + + /** Transform array transform to Substrait. */ + override def genArrayTransformTransformer( + substraitExprName: String, + argument: ExpressionTransformer, + function: ExpressionTransformer, + expr: ArrayTransform): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(argument, function), expr) + } + override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = generate override def genPostProjectForGenerate(generate: GenerateExec): SparkPlan = generate diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index 9327137fabe5..d3e3e9446036 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -713,4 +713,21 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS } } + + test("array functions with lambda") { + withTable("tb_array") { + sql("create table tb_array(ids array) using parquet") + sql(""" + |insert into tb_array values (array(1,5,2,null, 3)), (array(1,1,3,2)), (null), (array()) + |""".stripMargin) + val transform_sql = "select transform(ids, x -> x + 1) from tb_array" + runQueryAndCompare(transform_sql)(checkGlutenOperatorMatch[ProjectExecTransformer]) + + val filter_sql = "select filter(ids, x -> x % 2 == 1) from tb_array"; + runQueryAndCompare(filter_sql)(checkGlutenOperatorMatch[ProjectExecTransformer]) + + val aggregate_sql = "select ids, aggregate(ids, 3, (acc, x) -> acc + x) from tb_array"; + runQueryAndCompare(aggregate_sql)(checkGlutenOperatorMatch[ProjectExecTransformer]) + } + } } diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 77819fd73e75..ea33dc21080f 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -564,6 +564,16 @@ NamesAndTypesList SerializedPlanParser::blockToNameAndTypeList(const Block & hea return types; } +std::optional SerializedPlanParser::getFunctionSignatureName(UInt32 function_ref) const +{ + auto it = function_mapping.find(std::to_string(function_ref)); + if (it == function_mapping.end()) + return {}; + auto function_signature = it->second; + auto pos = function_signature.find(':'); + return function_signature.substr(0, pos); +} + std::string SerializedPlanParser::getFunctionName(const std::string & function_signature, const substrait::Expression_ScalarFunction & function) { @@ -1122,8 +1132,7 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionArgument( { std::string arg_name; bool keep_arg = FUNCTION_NEED_KEEP_ARGUMENTS.contains(function_name); - parseFunctionWithDAG(arg.value(), arg_name, actions_dag, keep_arg); - res = &actions_dag->getNodes().back(); + res = parseFunctionWithDAG(arg.value(), arg_name, actions_dag, keep_arg); } else { diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index ad2b0d50ec6a..ffd4148038cc 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -307,9 +307,12 @@ class SerializedPlanParser const std::unordered_map & getFunctionMapping() { return function_mapping; } static std::string getFunctionName(const std::string & function_sig, const substrait::Expression_ScalarFunction & function); + std::optional getFunctionSignatureName(UInt32 function_ref) const; IQueryPlanStep * addRemoveNullableStep(QueryPlan & plan, const std::set & columns); IQueryPlanStep * addRollbackFilterHeaderStep(QueryPlanPtr & query_plan, const Block & input_header); + + static std::pair parseLiteral(const substrait::Expression_Literal & literal); static ContextMutablePtr global_context; static Context::ConfigurationPtr config; @@ -384,7 +387,6 @@ class SerializedPlanParser // remove nullable after isNotNull void removeNullableForRequiredColumns(const std::set & require_columns, const ActionsDAGPtr & actions_dag) const; std::string getUniqueName(const std::string & name) { return name + "_" + std::to_string(name_no++); } - static std::pair parseLiteral(const substrait::Expression_Literal & literal); void wrapNullable( const std::vector & columns, ActionsDAGPtr actions_dag, std::map & nullable_measure_names); static std::pair convertStructFieldType(const DB::DataTypePtr & type, const DB::Field & field); diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp new file mode 100644 index 000000000000..584bc0ef1e04 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace local_engine +{ +class ArrayFilter : public FunctionParser +{ +public: + static constexpr auto name = "filter"; + explicit ArrayFilter(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~ArrayFilter() override = default; + + String getName() const override { return name; } + + String getCHFunctionName(const substrait::Expression_ScalarFunction & scalar_function) const override + { + return "arrayFilter"; + } + + const DB::ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, + DB::ActionsDAGPtr & actions_dag) const + { + auto ch_func_name = getCHFunctionName(substrait_func); + auto parsed_args = parseFunctionArguments(substrait_func, ch_func_name, actions_dag); + assert(parsed_args.size() == 2); + if (collectLambdaArguments(*plan_parser, substrait_func.arguments()[1].value().scalar_function()).size() == 1) + return toFunctionNode(actions_dag, ch_func_name, {parsed_args[1], parsed_args[0]}); + + /// filter with index argument. + const auto * range_end_node = toFunctionNode(actions_dag, "length", {toFunctionNode(actions_dag, "assumeNotNull", {parsed_args[0]})}); + range_end_node = ActionsDAGUtil::convertNodeType( + actions_dag, range_end_node, "Nullable(Int32)", range_end_node->result_name); + const auto * index_array_node = toFunctionNode( + actions_dag, + "range", + {addColumnToActionsDAG(actions_dag, std::make_shared(), 0), range_end_node}); + return toFunctionNode(actions_dag, ch_func_name, {parsed_args[1], parsed_args[0], index_array_node}); + } +}; +static FunctionParserRegister register_array_filter; + +class ArrayTransform : public FunctionParser +{ +public: + static constexpr auto name = "transform"; + explicit ArrayTransform(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~ArrayTransform() override = default; + String getName() const override { return name; } + String getCHFunctionName(const substrait::Expression_ScalarFunction & scalar_function) const override + { + return "arrayMap"; + } + + const DB::ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, + DB::ActionsDAGPtr & actions_dag) const + { + auto ch_func_name = getCHFunctionName(substrait_func); + auto lambda_args = collectLambdaArguments(*plan_parser, substrait_func.arguments()[1].value().scalar_function()); + auto parsed_args = parseFunctionArguments(substrait_func, ch_func_name, actions_dag); + assert(parsed_args.size() == 2); + if (lambda_args.size() == 1) + { + return toFunctionNode(actions_dag, ch_func_name, {parsed_args[1], parsed_args[0]}); + } + + /// transform with index argument. + const auto * range_end_node = toFunctionNode(actions_dag, "length", {toFunctionNode(actions_dag, "assumeNotNull", {parsed_args[0]})}); + range_end_node = ActionsDAGUtil::convertNodeType( + actions_dag, range_end_node, "Nullable(Int32)", range_end_node->result_name); + const auto * index_array_node = toFunctionNode( + actions_dag, + "range", + {addColumnToActionsDAG(actions_dag, std::make_shared(), 0), range_end_node}); + return toFunctionNode(actions_dag, ch_func_name, {parsed_args[1], parsed_args[0], index_array_node}); + } +}; +static FunctionParserRegister register_array_map; + +class ArrayAggregate : public FunctionParser +{ +public: + static constexpr auto name = "aggregate"; + explicit ArrayAggregate(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~ArrayAggregate() override = default; + String getName() const override { return name; } + String getCHFunctionName(const substrait::Expression_ScalarFunction & scalar_function) const override + { + return "arrayFold"; + } + const DB::ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, + DB::ActionsDAGPtr & actions_dag) const + { + auto ch_func_name = getCHFunctionName(substrait_func); + auto parsed_args = parseFunctionArguments(substrait_func, ch_func_name, actions_dag); + assert(parsed_args.size() == 3); + const auto * function_type = typeid_cast(parsed_args[2]->result_type.get()); + if (!function_type) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "The third argument of aggregate function must be a lambda function"); + if (!parsed_args[1]->result_type->equals(*(function_type->getReturnType()))) + { + parsed_args[1] = ActionsDAGUtil::convertNodeType( + actions_dag, + parsed_args[1], + function_type->getReturnType()->getName(), + parsed_args[1]->result_name); + } + + /// arrayFold cannot accept nullable(array) + const auto * array_col_node = parsed_args[0]; + if (parsed_args[0]->result_type->isNullable()) + { + array_col_node = toFunctionNode(actions_dag, "assumeNotNull", {parsed_args[0]}); + } + const auto * func_node = toFunctionNode(actions_dag, ch_func_name, {parsed_args[2], array_col_node, parsed_args[1]}); + /// For null array, result is null. + /// TODO: make a new version of arrayFold that can handle nullable array. + const auto * is_null_node = toFunctionNode(actions_dag, "isNull", {parsed_args[0]}); + const auto * null_node = addColumnToActionsDAG(actions_dag, DB::makeNullable(func_node->result_type), DB::Null()); + return toFunctionNode(actions_dag, "if", {is_null_node, null_node, func_node}); + } +}; +static FunctionParserRegister register_array_aggregate; + +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp new file mode 100644 index 000000000000..57c076ed2670 --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace local_engine +{ +DB::NamesAndTypesList collectLambdaArguments(const SerializedPlanParser & plan_parser_, const substrait::Expression_ScalarFunction & substrait_func) +{ + DB::NamesAndTypesList lambda_arguments; + std::unordered_set collected_names; + + for (const auto & arg : substrait_func.arguments()) + { + if (arg.value().has_scalar_function() + && plan_parser_.getFunctionSignatureName(arg.value().scalar_function().function_reference()) == "namedlambdavariable") + { + auto [_, col_name_field] = plan_parser_.parseLiteral(arg.value().scalar_function().arguments()[0].value().literal()); + String col_name = col_name_field.get(); + if (collected_names.contains(col_name)) + { + continue; + } + collected_names.insert(col_name); + auto type = TypeParser::parseType(arg.value().scalar_function().output_type()); + lambda_arguments.emplace_back(col_name, type); + } + } + return lambda_arguments; +} + +/// Refer to `PlannerActionsVisitorImpl::visitLambda` for how to build a lambda function node. +class LambdaFunction : public FunctionParser +{ +public: + static constexpr auto name = "lambdafunction"; + explicit LambdaFunction(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~LambdaFunction() override = default; + + String getName() const override { return name; } +protected: + String getCHFunctionName(const substrait::Expression_ScalarFunction & scalar_function) const override + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "getCHFunctionName is not implemented for LambdaFunction"); + } + + DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments( + const substrait::Expression_ScalarFunction & substrait_func, + const String & ch_func_name, + DB::ActionsDAGPtr & actions_dag) const override + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "parseFunctionArguments is not implemented for LambdaFunction"); + } + + const DB::ActionsDAG::Node * convertNodeTypeIfNeeded( + const substrait::Expression_ScalarFunction & substrait_func, + const DB::ActionsDAG::Node * func_node, + DB::ActionsDAGPtr & actions_dag) const override + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "convertNodeTypeIfNeeded is not implemented for NamedLambdaVariable"); + } + + const DB::ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, DB::ActionsDAGPtr & actions_dag) const override + { + /// Some special cases, for example, `transform(arr, x -> concat(arr, array(x)))` refers to + /// a column `arr` out of it directly. We need a `arr` as an input column for `lambda_actions_dag` + DB::NamesAndTypesList parent_header; + for (const auto * output_node : actions_dag->getOutputs()) + { + parent_header.emplace_back(output_node->result_name, output_node->result_type); + } + auto lambda_actions_dag = std::make_shared(parent_header); + + /// The first argument is the lambda function body, followings are the lambda arguments which is + /// needed by the lambda function body. + /// There could be a nested lambda function in the lambda function body, and it refer a variable from + /// this outside lambda function's arguments. For an example, transform(number, x -> transform(letter, y -> struct(x, y))). + /// Before parsing the lambda function body, we add lambda function arguments int actions dag at first. + for (size_t i = 1; i < substrait_func.arguments().size(); ++i) + { + (void)parseExpression(lambda_actions_dag, substrait_func.arguments()[i].value()); + } + const auto & substrait_lambda_body = substrait_func.arguments()[0].value(); + const auto * lambda_body_node = parseExpression(lambda_actions_dag, substrait_lambda_body); + lambda_actions_dag->getOutputs().push_back(lambda_body_node); + lambda_actions_dag->removeUnusedActions(Names(1, lambda_body_node->result_name)); + + auto expression_actions_settings = DB::ExpressionActionsSettings::fromContext(getContext(), DB::CompileExpressions::yes); + auto lambda_actions = std::make_shared(lambda_actions_dag, expression_actions_settings); + + DB::Names captured_column_names; + DB::Names required_column_names = lambda_actions->getRequiredColumns(); + DB::ActionsDAG::NodeRawConstPtrs lambda_children; + auto lambda_function_args = collectLambdaArguments(*plan_parser, substrait_func); + const auto & lambda_actions_inputs = lambda_actions_dag->getInputs(); + + std::unordered_map parent_nodes; + for (const auto & node : actions_dag->getNodes()) + { + parent_nodes[node.result_name] = &node; + } + for (const auto & required_column_name : required_column_names) + { + if (std::find_if( + lambda_function_args.begin(), + lambda_function_args.end(), + [&required_column_name](const DB::NameAndTypePair & name_type) { return name_type.name == required_column_name; }) + == lambda_function_args.end()) + { + auto it = std::find_if( + lambda_actions_inputs.begin(), + lambda_actions_inputs.end(), + [&required_column_name](const auto & node) { return node->result_name == required_column_name; }); + if (it == lambda_actions_inputs.end()) + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Required column not found: {}", required_column_name); + } + auto parent_node_it = parent_nodes.find(required_column_name); + if (parent_node_it == parent_nodes.end()) + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Not found column {} in actions dag:\n{}", + required_column_name, + actions_dag->dumpDAG()); + } + /// The nodes must be the ones in `actions_dag`, otherwise `ActionsDAG::evaluatePartialResult` will fail. Because nodes may have the + /// same name but their addresses are different. + lambda_children.push_back(parent_node_it->second); + captured_column_names.push_back(required_column_name); + } + } + + auto function_capture = std::make_shared( + lambda_actions, + captured_column_names, + lambda_function_args, + lambda_body_node->result_type, + lambda_body_node->result_name); + + const auto * result = &actions_dag->addFunction(function_capture, lambda_children, lambda_body_node->result_name); + return result; + } +}; + +static FunctionParserRegister register_lambda_function; + + +class NamedLambdaVariable : public FunctionParser +{ +public: + static constexpr auto name = "namedlambdavariable"; + explicit NamedLambdaVariable(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~NamedLambdaVariable() override = default; + + String getName() const override { return name; } +protected: + String getCHFunctionName(const substrait::Expression_ScalarFunction & scalar_function) const override + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "getCHFunctionName is not implemented for NamedLambdaVariable"); + } + + DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments( + const substrait::Expression_ScalarFunction & substrait_func, + const String & ch_func_name, + DB::ActionsDAGPtr & actions_dag) const override + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "parseFunctionArguments is not implemented for NamedLambdaVariable"); + } + + const DB::ActionsDAG::Node * convertNodeTypeIfNeeded( + const substrait::Expression_ScalarFunction & substrait_func, + const DB::ActionsDAG::Node * func_node, + DB::ActionsDAGPtr & actions_dag) const override + { + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "convertNodeTypeIfNeeded is not implemented for NamedLambdaVariable"); + } + + const DB::ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, DB::ActionsDAGPtr & actions_dag) const override + { + auto [_, col_name_field] = parseLiteral(substrait_func.arguments()[0].value().literal()); + String col_name = col_name_field.get(); + + auto type = TypeParser::parseType(substrait_func.output_type()); + const auto & inputs = actions_dag->getInputs(); + auto it = std::find_if(inputs.begin(), inputs.end(), [&col_name](const auto * node) { return node->result_name == col_name; }); + if (it == inputs.end()) + { + return &(actions_dag->addInput(col_name, type)); + } + return *it; + } +}; + +static FunctionParserRegister register_named_lambda_variable; + +} diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.h b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.h new file mode 100644 index 000000000000..327c72ade47c --- /dev/null +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.h @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +namespace local_engine +{ +DB::NamesAndTypesList collectLambdaArguments(const SerializedPlanParser & plan_parser_, const substrait::Expression_ScalarFunction & substrait_func); +} From 457898b08ac4aa41739898f9042d65176b17785f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 2 Jul 2024 15:02:16 +0800 Subject: [PATCH 380/402] [VL] IndicatorVectorPool to avoid sharing native columnar batches' ownerships among runtime instances (#6293) --- .../execution/RowToVeloxColumnarExec.scala | 2 +- .../ColumnarCachedBatchSerializer.scala | 2 +- .../datasources/velox/VeloxBlockStripes.java | 7 +- .../columnarbatch/ColumnarBatchTest.java | 104 ++++++++++++++++++ .../gluten/test/VeloxBackendTestBase.java | 86 +++++++++++++++ .../gluten/utils/VeloxBloomFilterTest.java | 68 +----------- cpp/core/jni/JniWrapper.cc | 12 -- ...lebornHashBasedColumnarShuffleWriter.scala | 7 -- .../gluten/memory/memtarget/Spillers.java | 3 + .../ColumnarBatchJniWrapper.java | 2 - .../gluten/columnarbatch/ColumnarBatches.java | 18 +-- .../gluten/columnarbatch/IndicatorVector.java | 16 ++- .../columnarbatch/IndicatorVectorBase.java | 33 +----- .../columnarbatch/IndicatorVectorPool.java | 66 +++++++++++ .../vectorized/ColumnarBatchOutIterator.java | 2 +- .../execution/ColumnarBuildSideRelation.scala | 4 +- .../spark/sql/execution/utils/ExecUtil.scala | 5 +- .../VeloxUniffleColumnarShuffleWriter.java | 7 -- 18 files changed, 300 insertions(+), 144 deletions(-) create mode 100644 backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java create mode 100644 backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java create mode 100644 gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 289df1a6e54d..7bcf56f7edb0 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -216,7 +216,7 @@ object RowToVeloxColumnarExec { try { val handle = jniWrapper .nativeConvertRowToColumnar(r2cHandle, rowLength.toArray, arrowBuf.memoryAddress()) - val cb = ColumnarBatches.create(runtime, handle) + val cb = ColumnarBatches.create(handle) convertTime += System.currentTimeMillis() - startNative cb } finally { diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index 3f82f919b4d8..15fd51abef48 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -247,7 +247,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe val batchHandle = jniWrapper .deserialize(deserializerHandle, cachedBatch.bytes) - val batch = ColumnarBatches.create(runtime, batchHandle) + val batch = ColumnarBatches.create(batchHandle) if (shouldSelectAttributes) { try { ColumnarBatches.select(batch, requestedColumnIndices.toArray) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java index 56df7b9ad57f..f9848d4ab634 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java @@ -16,13 +16,11 @@ */ package org.apache.spark.sql.execution.datasources.velox; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.columnarbatch.ColumnarBatches; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.UnsafeRow; import org.apache.spark.sql.execution.datasources.BlockStripe; import org.apache.spark.sql.execution.datasources.BlockStripes; -import org.apache.gluten.columnarbatch.ColumnarBatches; - import org.apache.spark.sql.vectorized.ColumnarBatch; import org.jetbrains.annotations.NotNull; @@ -53,8 +51,7 @@ public BlockStripe next() { return new BlockStripe() { @Override public ColumnarBatch getColumnarBatch() { - return ColumnarBatches.create( - Runtimes.contextInstance("VeloxBlockStripes"), blockAddresses[0]); + return ColumnarBatches.create(blockAddresses[0]); } @Override diff --git a/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java b/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java new file mode 100644 index 000000000000..cd2ac50d350c --- /dev/null +++ b/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.columnarbatch; + +import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; +import org.apache.gluten.test.VeloxBackendTestBase; +import org.apache.gluten.vectorized.ArrowWritableColumnVector; + +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnarBatch; +import org.apache.spark.util.TaskResources$; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.StreamSupport; + +public class ColumnarBatchTest extends VeloxBackendTestBase { + + @Test + public void testOffloadAndLoad() { + TaskResources$.MODULE$.runUnsafe( + () -> { + final int numRows = 100; + final ColumnarBatch batch = newArrowBatch("a boolean, b int", numRows); + Assert.assertTrue(ColumnarBatches.isHeavyBatch(batch)); + final ColumnarBatch offloaded = + ColumnarBatches.ensureOffloaded(ArrowBufferAllocators.contextInstance(), batch); + Assert.assertTrue(ColumnarBatches.isLightBatch(offloaded)); + final ColumnarBatch loaded = + ColumnarBatches.ensureLoaded(ArrowBufferAllocators.contextInstance(), offloaded); + Assert.assertTrue(ColumnarBatches.isHeavyBatch(loaded)); + long cnt = + StreamSupport.stream( + Spliterators.spliteratorUnknownSize( + loaded.rowIterator(), Spliterator.ORDERED), + false) + .count(); + Assert.assertEquals(numRows, cnt); + loaded.close(); + return null; + }); + } + + @Test + public void testCreateByHandle() { + TaskResources$.MODULE$.runUnsafe( + () -> { + final int numRows = 100; + final ColumnarBatch batch = newArrowBatch("a boolean, b int", numRows); + Assert.assertEquals(1, ColumnarBatches.getRefCnt(batch)); + final ColumnarBatch offloaded = + ColumnarBatches.ensureOffloaded(ArrowBufferAllocators.contextInstance(), batch); + Assert.assertEquals(1, ColumnarBatches.getRefCnt(offloaded)); + final long handle = ColumnarBatches.getNativeHandle(offloaded); + final ColumnarBatch created = ColumnarBatches.create(handle); + Assert.assertEquals(handle, ColumnarBatches.getNativeHandle(created)); + Assert.assertEquals(1, ColumnarBatches.getRefCnt(offloaded)); + Assert.assertEquals(1, ColumnarBatches.getRefCnt(created)); + ColumnarBatches.retain(created); + Assert.assertEquals(2, ColumnarBatches.getRefCnt(offloaded)); + Assert.assertEquals(2, ColumnarBatches.getRefCnt(created)); + ColumnarBatches.retain(offloaded); + Assert.assertEquals(3, ColumnarBatches.getRefCnt(offloaded)); + Assert.assertEquals(3, ColumnarBatches.getRefCnt(created)); + created.close(); + Assert.assertEquals(2, ColumnarBatches.getRefCnt(offloaded)); + Assert.assertEquals(2, ColumnarBatches.getRefCnt(created)); + offloaded.close(); + Assert.assertEquals(1, ColumnarBatches.getRefCnt(offloaded)); + Assert.assertEquals(1, ColumnarBatches.getRefCnt(created)); + created.close(); + Assert.assertEquals(0, ColumnarBatches.getRefCnt(offloaded)); + Assert.assertEquals(0, ColumnarBatches.getRefCnt(created)); + return null; + }); + } + + private static ColumnarBatch newArrowBatch(String schema, int numRows) { + final ArrowWritableColumnVector[] columns = + ArrowWritableColumnVector.allocateColumns(numRows, StructType.fromDDL(schema)); + for (ArrowWritableColumnVector col : columns) { + col.setValueCount(numRows); + } + final ColumnarBatch batch = new ColumnarBatch(columns); + batch.setNumRows(numRows); + return batch; + } +} diff --git a/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java b/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java new file mode 100644 index 000000000000..1d7df23566df --- /dev/null +++ b/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.test; + +import org.apache.gluten.GlutenConfig; +import org.apache.gluten.backendsapi.ListenerApi; +import org.apache.gluten.backendsapi.velox.VeloxListenerApi; + +import com.codahale.metrics.MetricRegistry; +import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; +import org.apache.spark.api.plugin.PluginContext; +import org.apache.spark.resource.ResourceInformation; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Map; + +/** For testing Velox backend without starting a Spark context. */ +public abstract class VeloxBackendTestBase { + @BeforeClass + public static void setup() { + final ListenerApi api = new VeloxListenerApi(); + api.onDriverStart(mockSparkContext(), mockPluginContext()); + } + + private static SparkContext mockSparkContext() { + // Not yet implemented. + return null; + } + + private static PluginContext mockPluginContext() { + return new PluginContext() { + @Override + public MetricRegistry metricRegistry() { + throw new UnsupportedOperationException(); + } + + @Override + public SparkConf conf() { + final SparkConf conf = new SparkConf(); + conf.set(GlutenConfig.GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY(), "0"); + return conf; + } + + @Override + public String executorID() { + throw new UnsupportedOperationException(); + } + + @Override + public String hostname() { + throw new UnsupportedOperationException(); + } + + @Override + public Map resources() { + throw new UnsupportedOperationException(); + } + + @Override + public void send(Object message) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Object ask(Object message) throws Exception { + throw new UnsupportedOperationException(); + } + }; + } +} diff --git a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java index fda4003ddd20..cf568b166582 100644 --- a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java @@ -16,34 +16,18 @@ */ package org.apache.gluten.utils; -import org.apache.gluten.GlutenConfig; -import org.apache.gluten.backendsapi.ListenerApi; -import org.apache.gluten.backendsapi.velox.VeloxListenerApi; - -import com.codahale.metrics.MetricRegistry; -import org.apache.spark.SparkConf; -import org.apache.spark.SparkContext; -import org.apache.spark.api.plugin.PluginContext; -import org.apache.spark.resource.ResourceInformation; +import org.apache.gluten.test.VeloxBackendTestBase; + import org.apache.spark.util.TaskResources$; import org.apache.spark.util.sketch.BloomFilter; import org.apache.spark.util.sketch.IncompatibleMergeException; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.function.ThrowingRunnable; -import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Map; - -public class VeloxBloomFilterTest { - @BeforeClass - public static void setup() { - final ListenerApi api = new VeloxListenerApi(); - api.onDriverStart(mockSparkContext(), mockPluginContext()); - } +public class VeloxBloomFilterTest extends VeloxBackendTestBase { @Test public void testEmpty() { TaskResources$.MODULE$.runUnsafe( @@ -191,50 +175,4 @@ private static void checkFalsePositives(BloomFilter filter, int start) { Assert.assertTrue(negativeFalsePositives > 0); Assert.assertTrue(negativeFalsePositives < attemptCount); } - - private static SparkContext mockSparkContext() { - // Not yet implemented. - return null; - } - - private static PluginContext mockPluginContext() { - return new PluginContext() { - @Override - public MetricRegistry metricRegistry() { - throw new UnsupportedOperationException(); - } - - @Override - public SparkConf conf() { - final SparkConf conf = new SparkConf(); - conf.set(GlutenConfig.GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY(), "0"); - return conf; - } - - @Override - public String executorID() { - throw new UnsupportedOperationException(); - } - - @Override - public String hostname() { - throw new UnsupportedOperationException(); - } - - @Override - public Map resources() { - throw new UnsupportedOperationException(); - } - - @Override - public void send(Object message) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public Object ask(Object message) throws Exception { - throw new UnsupportedOperationException(); - } - }; - } } diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 23eea2db7ce6..ea5c9d271c92 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -750,18 +750,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra JNI_METHOD_END(kInvalidObjectHandle) } -JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_obtainOwnership( // NOLINT - JNIEnv* env, - jobject wrapper, - jlong batchHandle) { - JNI_METHOD_START - auto ctx = gluten::getRuntime(env, wrapper); - auto batch = ObjectStore::retrieve(batchHandle); - auto newHandle = ctx->saveObject(batch); - return newHandle; - JNI_METHOD_END(-1L) -} - JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_close( // NOLINT JNIEnv* env, jobject wrapper, diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala index b8e6513cf009..87b16c65bd09 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala @@ -116,13 +116,6 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V]( if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { return 0L } - if (nativeShuffleWriter == -1L) { - throw new IllegalStateException( - "Fatal: spill() called before a celeborn shuffle writer " + - "is created. This behavior should be" + - "optimized by moving memory " + - "allocations from make() to split()") - } logInfo(s"Gluten shuffle writer: Trying to push $size bytes of data") // fixme pass true when being called by self val pushed = diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java index 4477e2956db7..38ed88f57778 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java @@ -80,6 +80,9 @@ public void append(Spiller spiller) { public long spill(MemoryTarget self, Phase phase, final long size) { long remainingBytes = size; for (Spiller spiller : spillers) { + if (remainingBytes <= 0) { + break; + } remainingBytes -= spiller.spill(self, phase, remainingBytes); } return size - remainingBytes; diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java index a834e13a4348..e71e9d7bee1b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java @@ -48,8 +48,6 @@ public static ColumnarBatchJniWrapper create(Runtime runtime) { public native long select(long batch, int[] columnIndices); - public native long obtainOwnership(long batch); - public native void close(long batch); @Override diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java index fc3b56c1bcad..cb68e032dc5b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java @@ -24,6 +24,7 @@ import org.apache.gluten.utils.ImplicitClass; import org.apache.gluten.vectorized.ArrowWritableColumnVector; +import com.google.common.annotations.VisibleForTesting; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.CDataDictionaryProvider; @@ -94,10 +95,12 @@ private static void transferVectors(ColumnarBatch from, ColumnarBatch target) { if (target.numCols() != from.numCols()) { throw new IllegalStateException(); } - final ColumnVector[] vectors = (ColumnVector[]) FIELD_COLUMNS.get(target); + final ColumnVector[] newVectors = new ColumnVector[from.numCols()]; for (int i = 0; i < target.numCols(); i++) { - vectors[i] = from.column(i); + newVectors[i] = from.column(i); } + FIELD_COLUMNS.set(target, newVectors); + System.out.println(); } catch (IllegalAccessException e) { throw new GlutenException(e); } @@ -127,7 +130,7 @@ public static ColumnarBatch select(ColumnarBatch batch, int[] columnIndices) { final IndicatorVector iv = getIndicatorVector(batch); long outputBatchHandle = ColumnarBatchJniWrapper.create(runtime).select(iv.handle(), columnIndices); - return create(runtime, outputBatchHandle); + return create(outputBatchHandle); case HEAVY: return new ColumnarBatch( Arrays.stream(columnIndices).mapToObj(batch::column).toArray(ColumnVector[]::new), @@ -218,7 +221,7 @@ private static ColumnarBatch offload(BufferAllocator allocator, ColumnarBatch in long handle = ColumnarBatchJniWrapper.create(runtime) .createWithArrowArray(cSchema.memoryAddress(), cArray.memoryAddress()); - ColumnarBatch output = ColumnarBatches.create(runtime, handle); + ColumnarBatch output = ColumnarBatches.create(handle); // Follow input's reference count. This might be optimized using // automatic clean-up or once the extensibility of ColumnarBatch is enriched @@ -294,7 +297,8 @@ private static long getRefCntHeavy(ColumnarBatch input) { return refCnt; } - private static long getRefCnt(ColumnarBatch input) { + @VisibleForTesting + static long getRefCnt(ColumnarBatch input) { switch (identifyBatchType(input)) { case LIGHT: return getRefCntLight(input); @@ -348,8 +352,8 @@ private static ColumnarBatch create(IndicatorVector iv) { return new ColumnarBatch(columnVectors, numRows); } - public static ColumnarBatch create(Runtime runtime, long nativeHandle) { - return create(new IndicatorVector(runtime, nativeHandle)); + public static ColumnarBatch create(long nativeHandle) { + return create(IndicatorVector.obtain(nativeHandle)); } public static void retain(ColumnarBatch b) { diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java index 0ec5b78ce500..7fe87e95fa54 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java @@ -16,15 +16,24 @@ */ package org.apache.gluten.columnarbatch; -import org.apache.gluten.exec.Runtime; +import org.apache.spark.util.TaskResources; import java.util.concurrent.atomic.AtomicLong; public class IndicatorVector extends IndicatorVectorBase { + private final IndicatorVectorPool pool; private final AtomicLong refCnt = new AtomicLong(1L); - protected IndicatorVector(Runtime runtime, long handle) { - super(runtime, handle); + protected IndicatorVector(IndicatorVectorPool pool, long handle) { + super(handle); + this.pool = pool; + } + + static IndicatorVector obtain(long handle) { + final IndicatorVectorPool pool = + TaskResources.addResourceIfNotRegistered( + IndicatorVectorPool.class.getName(), IndicatorVectorPool::new); + return pool.obtain(handle); } @Override @@ -44,6 +53,7 @@ void release() { return; } if (refCnt.decrementAndGet() == 0) { + pool.remove(handle); jniWrapper.close(handle); } } diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java index fa695127adbf..1bc685bd5ceb 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.columnarbatch; -import org.apache.gluten.exec.Runtime; +import org.apache.gluten.exec.Runtimes; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Decimal; @@ -26,35 +26,14 @@ import org.apache.spark.unsafe.types.UTF8String; public abstract class IndicatorVectorBase extends ColumnVector { - private final Runtime runtime; - protected final long handle; protected final ColumnarBatchJniWrapper jniWrapper; + protected final long handle; - protected IndicatorVectorBase(Runtime runtime, long handle) { + protected IndicatorVectorBase(long handle) { super(DataTypes.NullType); - this.runtime = runtime; - this.jniWrapper = ColumnarBatchJniWrapper.create(runtime); - this.handle = takeOwnership(handle); - } - - private long takeOwnership(long handle) { - // Note: Underlying memory of returned batch still holds - // reference to the original memory manager. As - // a result, once its original resident runtime / mm is - // released, data may become invalid. Currently, it's - // the caller's responsibility to make sure the original - // runtime / mm keep alive even this function - // was called. - // - // Additionally, as in Gluten we have principle that runtime - // mm that were created earlier will be released - // later, this FILO practice is what helps the runtime that - // took ownership be able to access the data constantly - // because the original runtime will live longer than - // itself. - long newHandle = jniWrapper.obtainOwnership(handle); - jniWrapper.close(handle); - return newHandle; + this.jniWrapper = + ColumnarBatchJniWrapper.create(Runtimes.contextInstance("IndicatorVectorBase#init")); + this.handle = handle; } public String getType() { diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java new file mode 100644 index 000000000000..6e46742b564a --- /dev/null +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.columnarbatch; + +import org.apache.spark.util.TaskResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class IndicatorVectorPool implements TaskResource { + private static final Logger LOG = LoggerFactory.getLogger(IndicatorVectorPool.class); + // A pool for all alive indicator vectors. The reason we adopt the pool + // is, we don't want one native columnar batch (which is located via the + // long int handle through JNI bridge) to be owned by more than one IndicatorVector + // instance so release method of the native columnar batch could be guaranteed + // to be called and only called once. + private final Map uniqueInstances = new ConcurrentHashMap<>(); + + IndicatorVectorPool() {} + + @Override + public void release() throws Exception { + if (!uniqueInstances.isEmpty()) { + LOG.warn( + "There are still unreleased native columnar batches during ending the task." + + " Will close them automatically however the batches should be better released" + + " manually to minimize memory pressure."); + } + } + + IndicatorVector obtain(long handle) { + return uniqueInstances.computeIfAbsent(handle, h -> new IndicatorVector(this, handle)); + } + + void remove(long handle) { + if (uniqueInstances.remove(handle) == null) { + throw new IllegalStateException("Indicator vector not found in pool, this should not happen"); + } + } + + @Override + public int priority() { + return 0; + } + + @Override + public String resourceName() { + return IndicatorVectorPool.class.getName(); + } +} diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index ddf00844f9b0..9dd0404384ad 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -67,7 +67,7 @@ public ColumnarBatch nextInternal() throws IOException { if (batchHandle == -1L) { return null; // stream ended } - return ColumnarBatches.create(runtime, batchHandle); + return ColumnarBatches.create(batchHandle); } @Override diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index 2f6abdc370d6..f7bcfd694d52 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -68,7 +68,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra jniWrapper .deserialize(serializeHandle, batches(batchId)) batchId += 1 - ColumnarBatches.create(runtime, handle) + ColumnarBatches.create(handle) } }) .protectInvocationFlow() @@ -124,7 +124,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra batchId += 1 val batchHandle = serializerJniWrapper.deserialize(serializeHandle, batchBytes) - val batch = ColumnarBatches.create(runtime, batchHandle) + val batch = ColumnarBatches.create(batchHandle) if (batch.numRows == 0) { batch.close() Iterator.empty diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index 22b376a1b608..77f35ff48fcc 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -145,10 +145,7 @@ object ExecUtil { val newHandle = ColumnarBatches.compose(pidBatch, cb) // Composed batch already hold pidBatch's shared ref, so close is safe. ColumnarBatches.forceClose(pidBatch) - ( - 0, - ColumnarBatches - .create(Runtimes.contextInstance("ExecUtil#getShuffleDependency"), newHandle)) + (0, ColumnarBatches.create(newHandle)) }) .recyclePayload(p => ColumnarBatches.forceClose(p._2)) // FIXME why force close? .create() diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index bd205ba7a469..ca5b3ad9529f 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -165,13 +165,6 @@ public long spill(MemoryTarget self, Spiller.Phase phase, long size) { if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) { return 0L; } - if (nativeShuffleWriter == -1) { - throw new IllegalStateException( - "Fatal: spill() called before a shuffle shuffle writer " - + "evaluator is created. This behavior should be" - + "optimized by moving memory " - + "allocations from make() to split()"); - } LOG.info("Gluten shuffle writer: Trying to push {} bytes of data", size); long pushed = jniWrapper.nativeEvict(nativeShuffleWriter, size, false); LOG.info("Gluten shuffle writer: Pushed {} / {} bytes of data", pushed, size); From 264ff2ecd28dfecd054b77c78c1b7f2c0db982e2 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Tue, 2 Jul 2024 15:09:29 +0800 Subject: [PATCH 381/402] [CORE] Fix non-deterministic filter executed twice when push down to scan (#6296) Co-authored-by: wangguangxin.cn --- .../gluten/execution/TestOperator.scala | 19 +++++++++++++++++++ .../BasicPhysicalOperatorTransformer.scala | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index 9b47a519cd28..c010b9128ce1 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1892,4 +1892,23 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } } + + test("fix non-deterministic filter executed twice when push down to scan") { + val df = sql("select * from lineitem where rand() <= 0.5") + // plan check + val plan = df.queryExecution.executedPlan + val scans = plan.collect { case scan: FileSourceScanExecTransformer => scan } + val filters = plan.collect { case filter: FilterExecTransformer => filter } + assert(scans.size == 1) + assert(filters.size == 1) + assert(scans(0).dataFilters.size == 1) + val remainingFilters = FilterHandler.getRemainingFilters( + scans(0).dataFilters, + splitConjunctivePredicates(filters(0).condition)) + assert(remainingFilters.size == 0) + + // result length check, table lineitem has 60,000 rows + val resultLength = df.collect().length + assert(resultLength > 25000 && resultLength < 35000) + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala index 962ad6aca9d3..0b792d52e056 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala @@ -365,7 +365,7 @@ object FilterHandler extends PredicateHelper { * the filter conditions not pushed down into Scan. */ def getRemainingFilters(scanFilters: Seq[Expression], filters: Seq[Expression]): Seq[Expression] = - (ExpressionSet(filters) -- ExpressionSet(scanFilters)).toSeq + (filters.toSet -- scanFilters.toSet).toSeq // Separate and compare the filter conditions in Scan and Filter. // Try to push down the remaining conditions in Filter into Scan. From f9bd4900640dfd222065e9b6b22dd5a21be1bf7d Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 2 Jul 2024 16:25:02 +0800 Subject: [PATCH 382/402] [VL] Add isStreamingAgg info to HashAggregateTransformer (#6307) --- .../execution/HashAggregateExecBaseTransformer.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala index 49a9ee1e816a..9345b3a3636f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala @@ -84,9 +84,14 @@ abstract class HashAggregateExecBaseTransformer( val functionString = truncatedString(allAggregateExpressions, "[", ", ", "]", maxFields) val outputString = truncatedString(output, "[", ", ", "]", maxFields) if (verbose) { - s"HashAggregateTransformer(keys=$keyString, functions=$functionString, output=$outputString)" + s"HashAggregateTransformer(keys=$keyString, " + + s"functions=$functionString, " + + s"isStreamingAgg=$isCapableForStreamingAggregation, " + + s"output=$outputString)" } else { - s"HashAggregateTransformer(keys=$keyString, functions=$functionString)" + s"HashAggregateTransformer(keys=$keyString, " + + s"functions=$functionString, " + + s"isStreamingAgg=$isCapableForStreamingAggregation)" } } From dde87b15b13e8a1dd43b65822cb70709c1d4f429 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Tue, 2 Jul 2024 17:03:00 +0800 Subject: [PATCH 383/402] [GLUTEN-6279][CH] Inroduce JNI safe array (#6280) * jni safe array * return JString * better --- .../gluten/vectorized/BatchIterator.java | 7 +- cpp-ch/local-engine/Common/CHUtil.cpp | 4 +- cpp-ch/local-engine/Common/CHUtil.h | 4 +- cpp-ch/local-engine/Parser/RelMetric.cpp | 2 +- cpp-ch/local-engine/Parser/RelMetric.h | 2 +- .../Parser/SerializedPlanParser.cpp | 2 +- .../Parser/SerializedPlanParser.h | 6 +- cpp-ch/local-engine/jni/jni_common.cpp | 14 +- cpp-ch/local-engine/jni/jni_common.h | 99 ++++++++ cpp-ch/local-engine/local_engine_jni.cpp | 221 +++++++----------- 10 files changed, 199 insertions(+), 162 deletions(-) diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java index d674c6e90def..1fbb6053a2af 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/BatchIterator.java @@ -17,6 +17,7 @@ package org.apache.gluten.vectorized; import org.apache.gluten.metrics.IMetrics; +import org.apache.gluten.metrics.NativeMetrics; import org.apache.spark.sql.execution.utils.CHExecUtil; import org.apache.spark.sql.vectorized.ColumnVector; @@ -50,7 +51,7 @@ public String getId() { private native void nativeCancel(long nativeHandle); - private native IMetrics nativeFetchMetrics(long nativeHandle); + private native String nativeFetchMetrics(long nativeHandle); @Override public boolean hasNextInternal() throws IOException { @@ -72,8 +73,8 @@ public ColumnarBatch nextInternal() throws IOException { } @Override - public IMetrics getMetricsInternal() throws IOException, ClassNotFoundException { - return nativeFetchMetrics(handle); + public IMetrics getMetricsInternal() { + return new NativeMetrics(nativeFetchMetrics(handle)); } @Override diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 76c71ce752d6..4a21dbe39834 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -468,7 +468,7 @@ String QueryPipelineUtil::explainPipeline(DB::QueryPipeline & pipeline) using namespace DB; -std::map BackendInitializerUtil::getBackendConfMap(const std::string & plan) +std::map BackendInitializerUtil::getBackendConfMap(const std::string_view plan) { std::map ch_backend_conf; if (plan.empty()) @@ -972,7 +972,7 @@ void BackendInitializerUtil::init(const std::string & plan) }); } -void BackendInitializerUtil::updateConfig(const DB::ContextMutablePtr & context, const std::string & plan) +void BackendInitializerUtil::updateConfig(const DB::ContextMutablePtr & context, const std::string_view plan) { std::map backend_conf_map = getBackendConfMap(plan); diff --git a/cpp-ch/local-engine/Common/CHUtil.h b/cpp-ch/local-engine/Common/CHUtil.h index 1198cfa2195d..3ac0f63ce10b 100644 --- a/cpp-ch/local-engine/Common/CHUtil.h +++ b/cpp-ch/local-engine/Common/CHUtil.h @@ -141,7 +141,7 @@ class BackendInitializerUtil /// 1. global level resources like global_context/shared_context, notice that they can only be initialized once in process lifetime /// 2. session level resources like settings/configs, they can be initialized multiple times following the lifetime of executor/driver static void init(const std::string & plan); - static void updateConfig(const DB::ContextMutablePtr &, const std::string &); + static void updateConfig(const DB::ContextMutablePtr &, const std::string_view); // use excel text parser @@ -199,7 +199,7 @@ class BackendInitializerUtil static std::vector wrapDiskPathConfig(const String & path_prefix, const String & path_suffix, Poco::Util::AbstractConfiguration & config); - static std::map getBackendConfMap(const std::string & plan); + static std::map getBackendConfMap(const std::string_view plan); inline static std::once_flag init_flag; inline static Poco::Logger * logger; diff --git a/cpp-ch/local-engine/Parser/RelMetric.cpp b/cpp-ch/local-engine/Parser/RelMetric.cpp index eec31213a69e..feb930dfc4ed 100644 --- a/cpp-ch/local-engine/Parser/RelMetric.cpp +++ b/cpp-ch/local-engine/Parser/RelMetric.cpp @@ -142,7 +142,7 @@ const String & RelMetric::getName() const return name; } -std::string RelMetricSerializer::serializeRelMetric(RelMetricPtr rel_metric, bool flatten) +std::string RelMetricSerializer::serializeRelMetric(const RelMetricPtr & rel_metric, bool flatten) { StringBuffer result; Writer writer(result); diff --git a/cpp-ch/local-engine/Parser/RelMetric.h b/cpp-ch/local-engine/Parser/RelMetric.h index 8255654a8dde..8706bed2ff02 100644 --- a/cpp-ch/local-engine/Parser/RelMetric.h +++ b/cpp-ch/local-engine/Parser/RelMetric.h @@ -58,6 +58,6 @@ class RelMetric class RelMetricSerializer { public: - static std::string serializeRelMetric(RelMetricPtr rel_metric, bool flatten = true); + static std::string serializeRelMetric(const RelMetricPtr & rel_metric, bool flatten = true); }; } diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index ea33dc21080f..1ee485346d07 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -1742,7 +1742,7 @@ std::unique_ptr SerializedPlanParser::createExecutor(DB::QueryPla context, std::move(query_plan), std::move(pipeline), query_plan->getCurrentDataStream().header.cloneEmpty()); } -QueryPlanPtr SerializedPlanParser::parse(const std::string_view & plan) +QueryPlanPtr SerializedPlanParser::parse(const std::string_view plan) { substrait::Plan s_plan; /// https://stackoverflow.com/questions/52028583/getting-error-parsing-protobuf-data diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index ffd4148038cc..c62dc73c9394 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -258,7 +258,7 @@ class SerializedPlanParser std::unique_ptr createExecutor(DB::QueryPlanPtr query_plan); - DB::QueryPlanPtr parse(const std::string_view & plan); + DB::QueryPlanPtr parse(const std::string_view plan); DB::QueryPlanPtr parse(const substrait::Plan & plan); public: @@ -270,7 +270,7 @@ class SerializedPlanParser /// template - std::unique_ptr createExecutor(const std::string_view & plan); + std::unique_ptr createExecutor(const std::string_view plan); DB::QueryPlanStepPtr parseReadRealWithLocalFile(const substrait::ReadRel & rel); DB::QueryPlanStepPtr parseReadRealWithJavaIter(const substrait::ReadRel & rel); @@ -407,7 +407,7 @@ class SerializedPlanParser }; template -std::unique_ptr SerializedPlanParser::createExecutor(const std::string_view & plan) +std::unique_ptr SerializedPlanParser::createExecutor(const std::string_view plan) { return createExecutor(JsonPlan ? parseJson(plan) : parse(plan)); } diff --git a/cpp-ch/local-engine/jni/jni_common.cpp b/cpp-ch/local-engine/jni/jni_common.cpp index 4d05b5f48e58..6eb02a2f450b 100644 --- a/cpp-ch/local-engine/jni/jni_common.cpp +++ b/cpp-ch/local-engine/jni/jni_common.cpp @@ -73,13 +73,13 @@ jmethodID GetStaticMethodID(JNIEnv * env, jclass this_class, const char * name, jstring charTojstring(JNIEnv * env, const char * pat) { - jclass str_class = (env)->FindClass("Ljava/lang/String;"); - jmethodID ctor_id = (env)->GetMethodID(str_class, "", "([BLjava/lang/String;)V"); - jsize strSize = static_cast(strlen(pat)); - jbyteArray bytes = (env)->NewByteArray(strSize); - (env)->SetByteArrayRegion(bytes, 0, strSize, reinterpret_cast(const_cast(pat))); - jstring encoding = (env)->NewStringUTF("UTF-8"); - jstring result = static_cast((env)->NewObject(str_class, ctor_id, bytes, encoding)); + const jclass str_class = (env)->FindClass("Ljava/lang/String;"); + const jmethodID ctor_id = (env)->GetMethodID(str_class, "", "([BLjava/lang/String;)V"); + const jsize str_size = static_cast(strlen(pat)); + const jbyteArray bytes = (env)->NewByteArray(str_size); + (env)->SetByteArrayRegion(bytes, 0, str_size, reinterpret_cast(const_cast(pat))); + const jstring encoding = (env)->NewStringUTF("UTF-8"); + const auto result = static_cast((env)->NewObject(str_class, ctor_id, bytes, encoding)); env->DeleteLocalRef(bytes); env->DeleteLocalRef(encoding); return result; diff --git a/cpp-ch/local-engine/jni/jni_common.h b/cpp-ch/local-engine/jni/jni_common.h index 8d14370835c4..c1e0fbead535 100644 --- a/cpp-ch/local-engine/jni/jni_common.h +++ b/cpp-ch/local-engine/jni/jni_common.h @@ -141,4 +141,103 @@ jlong safeCallStaticLongMethod(JNIEnv * env, jclass clazz, jmethodID method_id, LOCAL_ENGINE_JNI_JMETHOD_END(env) return ret; } + +// Safe version of JNI {Get|Release}ArrayElements routines. +// SafeNativeArray would release the managed array elements automatically +// during destruction. + +enum class JniPrimitiveArrayType { + kBoolean = 0, + kByte = 1, + kChar = 2, + kShort = 3, + kInt = 4, + kLong = 5, + kFloat = 6, + kDouble = 7 +}; + +#define CONCATENATE(t1, t2, t3) t1##t2##t3 + +#define DEFINE_PRIMITIVE_ARRAY(PRIM_TYPE, JAVA_TYPE, JNI_NATIVE_TYPE, NATIVE_TYPE, METHOD_VAR) \ + template <> \ + struct JniPrimitiveArray { \ + using JavaType = JAVA_TYPE; \ + using JniNativeType = JNI_NATIVE_TYPE; \ + using NativeType = NATIVE_TYPE; \ + \ + static JniNativeType get(JNIEnv* env, JavaType javaArray) { \ + return env->CONCATENATE(Get, METHOD_VAR, ArrayElements)(javaArray, nullptr); \ + } \ + \ + static void release(JNIEnv* env, JavaType javaArray, JniNativeType nativeArray) { \ + env->CONCATENATE(Release, METHOD_VAR, ArrayElements)(javaArray, nativeArray, JNI_ABORT); \ + } \ + }; + +template +struct JniPrimitiveArray {}; + +DEFINE_PRIMITIVE_ARRAY(kBoolean, jbooleanArray, jboolean*, bool*, Boolean) +DEFINE_PRIMITIVE_ARRAY(kByte, jbyteArray, jbyte*, uint8_t*, Byte) +DEFINE_PRIMITIVE_ARRAY(kChar, jcharArray, jchar*, uint16_t*, Char) +DEFINE_PRIMITIVE_ARRAY(kShort, jshortArray, jshort*, int16_t*, Short) +DEFINE_PRIMITIVE_ARRAY(kInt, jintArray, jint*, int32_t*, Int) +DEFINE_PRIMITIVE_ARRAY(kLong, jlongArray, jlong*, int64_t*, Long) +DEFINE_PRIMITIVE_ARRAY(kFloat, jfloatArray, jfloat*, float_t*, Float) +DEFINE_PRIMITIVE_ARRAY(kDouble, jdoubleArray, jdouble*, double_t*, Double) + +template +class SafeNativeArray { + using PrimitiveArray = JniPrimitiveArray; + using JavaArrayType = typename PrimitiveArray::JavaType; + using JniNativeArrayType = typename PrimitiveArray::JniNativeType; + using NativeArrayType = typename PrimitiveArray::NativeType; + + public: + virtual ~SafeNativeArray() { + PrimitiveArray::release(env_, javaArray_, nativeArray_); + } + + SafeNativeArray(const SafeNativeArray&) = delete; + SafeNativeArray(SafeNativeArray&&) = delete; + SafeNativeArray& operator=(const SafeNativeArray&) = delete; + SafeNativeArray& operator=(SafeNativeArray&&) = delete; + + const NativeArrayType elems() const { + return reinterpret_cast(nativeArray_); + } + + const jsize length() const { + return env_->GetArrayLength(javaArray_); + } + + static SafeNativeArray get(JNIEnv* env, JavaArrayType javaArray) { + JniNativeArrayType nativeArray = PrimitiveArray::get(env, javaArray); + return SafeNativeArray(env, javaArray, nativeArray); + } + + private: + SafeNativeArray(JNIEnv* env, JavaArrayType javaArray, JniNativeArrayType nativeArray) + : env_(env), javaArray_(javaArray), nativeArray_(nativeArray){}; + + JNIEnv* env_; + JavaArrayType javaArray_; + JniNativeArrayType nativeArray_; +}; + +#define DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(PRIM_TYPE, JAVA_TYPE, METHOD_VAR) \ + inline SafeNativeArray CONCATENATE(get, METHOD_VAR, ArrayElementsSafe)( \ + JNIEnv * env, JAVA_TYPE array) { \ + return SafeNativeArray::get(env, array); \ + } + +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kBoolean, jbooleanArray, Boolean) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kByte, jbyteArray, Byte) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kChar, jcharArray, Char) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kShort, jshortArray, Short) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kInt, jintArray, Int) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kLong, jlongArray, Long) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kFloat, jfloatArray, Float) +DEFINE_SAFE_GET_PRIMITIVE_ARRAY_FUNCTIONS(kDouble, jdoubleArray, Double) } diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 9c642d70ec27..2338bfe8b1e6 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -93,16 +93,6 @@ static std::string jstring2string(JNIEnv * env, jstring jStr) return ret; } -static jstring stringTojstring(JNIEnv * env, const char * pat) -{ - jclass strClass = (env)->FindClass("java/lang/String"); - jmethodID ctorID = (env)->GetMethodID(strClass, "", "([BLjava/lang/String;)V"); - jbyteArray bytes = (env)->NewByteArray(strlen(pat)); - (env)->SetByteArrayRegion(bytes, 0, strlen(pat), reinterpret_cast(pat)); - jstring encoding = (env)->NewStringUTF("UTF-8"); - return static_cast((env)->NewObject(strClass, ctorID, bytes, encoding)); -} - extern "C" { #endif @@ -121,9 +111,6 @@ static jmethodID block_stripes_constructor; static jclass split_result_class; static jmethodID split_result_constructor; -static jclass native_metrics_class; -static jmethodID native_metrics_constructor; - JNIEXPORT jint JNI_OnLoad(JavaVM * vm, void * /*reserved*/) { JNIEnv * env; @@ -188,10 +175,6 @@ JNIEXPORT jint JNI_OnLoad(JavaVM * vm, void * /*reserved*/) local_engine::ReservationListenerWrapper::reservation_listener_currentMemory = local_engine::GetMethodID(env, local_engine::ReservationListenerWrapper::reservation_listener_class, "currentMemory", "()J"); - - native_metrics_class = local_engine::CreateGlobalClassReference(env, "Lorg/apache/gluten/metrics/NativeMetrics;"); - native_metrics_constructor = local_engine::GetMethodID(env, native_metrics_class, "", "(Ljava/lang/String;)V"); - local_engine::BroadCastJoinBuilder::init(env); local_engine::JNIUtils::vm = vm; @@ -218,16 +201,14 @@ JNIEXPORT void JNI_OnUnload(JavaVM * vm, void * /*reserved*/) env->DeleteGlobalRef(local_engine::SourceFromJavaIter::serialized_record_batch_iterator_class); env->DeleteGlobalRef(local_engine::SparkRowToCHColumn::spark_row_interator_class); env->DeleteGlobalRef(local_engine::ReservationListenerWrapper::reservation_listener_class); - env->DeleteGlobalRef(native_metrics_class); } JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeInitNative(JNIEnv * env, jobject, jbyteArray conf_plan) { LOCAL_ENGINE_JNI_METHOD_START - std::string::size_type plan_buf_size = env->GetArrayLength(conf_plan); - jbyte * plan_buf_addr = env->GetByteArrayElements(conf_plan, nullptr); - local_engine::BackendInitializerUtil::init({reinterpret_cast(plan_buf_addr), plan_buf_size}); - env->ReleaseByteArrayElements(conf_plan, plan_buf_addr, JNI_ABORT); + const auto conf_plan_a = local_engine::getByteArrayElementsSafe(env, conf_plan); + const std::string::size_type plan_buf_size = conf_plan_a.length(); + local_engine::BackendInitializerUtil::init({reinterpret_cast(conf_plan_a.elems()), plan_buf_size}); LOCAL_ENGINE_JNI_METHOD_END(env, ) } @@ -252,9 +233,10 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ auto query_context = local_engine::getAllocator(allocator_id)->query_context; // by task update new configs ( in case of dynamic config update ) - std::string::size_type plan_buf_size = env->GetArrayLength(conf_plan); - jbyte * plan_buf_addr = env->GetByteArrayElements(conf_plan, nullptr); - local_engine::BackendInitializerUtil::updateConfig(query_context, {reinterpret_cast(plan_buf_addr), plan_buf_size}); + const auto conf_plan_a = local_engine::getByteArrayElementsSafe(env, conf_plan); + const std::string::size_type conf_plan_size = conf_plan_a.length(); + local_engine::BackendInitializerUtil::updateConfig( + query_context, {reinterpret_cast(conf_plan_a.elems()), conf_plan_size}); local_engine::SerializedPlanParser parser(query_context); jsize iter_num = env->GetArrayLength(iter_arr); @@ -268,21 +250,20 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ for (jsize i = 0, split_info_arr_size = env->GetArrayLength(split_infos); i < split_info_arr_size; i++) { jbyteArray split_info = static_cast(env->GetObjectArrayElement(split_infos, i)); - std::string::size_type split_info_size = env->GetArrayLength(split_info); - jbyte * split_info_addr = env->GetByteArrayElements(split_info, nullptr); - parser.addSplitInfo(std::string{reinterpret_cast(split_info_addr), split_info_size}); + const auto split_info_a = local_engine::getByteArrayElementsSafe(env, split_info); + const std::string::size_type split_info_size = split_info_a.length(); + parser.addSplitInfo({reinterpret_cast(split_info_a.elems()), split_info_size}); } - std::string::size_type plan_size = env->GetArrayLength(plan); - jbyte * plan_address = env->GetByteArrayElements(plan, nullptr); + const auto plan_a = local_engine::getByteArrayElementsSafe(env, plan); + const std::string::size_type plan_size = plan_a.length(); local_engine::LocalExecutor * executor - = parser.createExecutor({reinterpret_cast(plan_address), plan_size}).release(); + = parser.createExecutor({reinterpret_cast(plan_a.elems()), plan_size}).release(); local_engine::LocalExecutor::addExecutor(executor); LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); executor->setMetric(parser.getMetric()); executor->setExtraPlanHolder(parser.extra_plan_holder); - env->ReleaseByteArrayElements(plan, plan_address, JNI_ABORT); - env->ReleaseByteArrayElements(conf_plan, plan_buf_addr, JNI_ABORT); + return reinterpret_cast(executor); LOCAL_ENGINE_JNI_METHOD_END(env, -1) } @@ -309,7 +290,7 @@ JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeCancel(JNIE { LOCAL_ENGINE_JNI_METHOD_START local_engine::LocalExecutor::removeExecutor(executor_address); - local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); + auto *executor = reinterpret_cast(executor_address); executor->cancel(); LOG_INFO(&Poco::Logger::get("jni"), "Cancel LocalExecutor {}", reinterpret_cast(executor)); LOCAL_ENGINE_JNI_METHOD_END(env, ) @@ -319,22 +300,21 @@ JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeClose(JNIEn { LOCAL_ENGINE_JNI_METHOD_START local_engine::LocalExecutor::removeExecutor(executor_address); - local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); + auto *executor = reinterpret_cast(executor_address); LOG_INFO(&Poco::Logger::get("jni"), "Finalize LocalExecutor {}", reinterpret_cast(executor)); delete executor; LOCAL_ENGINE_JNI_METHOD_END(env, ) } -JNIEXPORT jobject Java_org_apache_gluten_vectorized_BatchIterator_nativeFetchMetrics(JNIEnv * env, jobject /*obj*/, jlong executor_address) +JNIEXPORT jstring Java_org_apache_gluten_vectorized_BatchIterator_nativeFetchMetrics(JNIEnv * env, jobject /*obj*/, jlong executor_address) { LOCAL_ENGINE_JNI_METHOD_START /// Collect metrics only if optimizations are disabled, otherwise coredump would happen. - local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); - auto metric = executor->getMetric(); - String metrics_json = metric ? local_engine::RelMetricSerializer::serializeRelMetric(metric) : ""; - LOG_DEBUG(&Poco::Logger::get("jni"), "{}", metrics_json); - jobject native_metrics = env->NewObject(native_metrics_class, native_metrics_constructor, stringTojstring(env, metrics_json.c_str())); - return native_metrics; + const local_engine::LocalExecutor * executor = reinterpret_cast(executor_address); + const auto metric = executor->getMetric(); + const String metrics_json = metric ? local_engine::RelMetricSerializer::serializeRelMetric(metric) : ""; + + return local_engine::charTojstring(env, metrics_json.c_str()); LOCAL_ENGINE_JNI_METHOD_END(env, nullptr) } @@ -584,22 +564,16 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na std::string out_exprs; if (expr_list != nullptr) { - int len = env->GetArrayLength(expr_list); - auto * str = reinterpret_cast(new char[len]); - memset(str, 0, len); - env->GetByteArrayRegion(expr_list, 0, len, str); - hash_exprs = std::string(str, str + len); - delete[] str; + const auto expr_list_a = local_engine::getByteArrayElementsSafe(env, expr_list); + const std::string::size_type expr_list_size = expr_list_a.length(); + hash_exprs = std::string{reinterpret_cast(expr_list_a.elems()), expr_list_size}; } if (out_expr_list != nullptr) { - int len = env->GetArrayLength(out_expr_list); - auto * str = reinterpret_cast(new char[len]); - memset(str, 0, len); - env->GetByteArrayRegion(out_expr_list, 0, len, str); - out_exprs = std::string(str, str + len); - delete[] str; + const auto out_expr_list_a = local_engine::getByteArrayElementsSafe(env, out_expr_list); + const std::string::size_type out_expr_list_size = out_expr_list_a.length(); + out_exprs = std::string{reinterpret_cast(out_expr_list_a.elems()), out_expr_list_size}; } Poco::StringTokenizer local_dirs_tokenizer(jstring2string(env, local_dirs), ","); @@ -660,20 +634,16 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_CHShuffleSplitterJniWrapper_na std::string out_exprs; if (expr_list != nullptr) { - int len = env->GetArrayLength(expr_list); - auto * str = reinterpret_cast(new char[len]); - env->GetByteArrayRegion(expr_list, 0, len, str); - hash_exprs = std::string(str, str + len); - delete[] str; + const auto expr_list_a = local_engine::getByteArrayElementsSafe(env, expr_list); + const std::string::size_type expr_list_size = expr_list_a.length(); + hash_exprs = std::string{reinterpret_cast(expr_list_a.elems()), expr_list_size}; } if (out_expr_list != nullptr) { - int len = env->GetArrayLength(out_expr_list); - auto * str = reinterpret_cast(new char[len]); - env->GetByteArrayRegion(out_expr_list, 0, len, str); - out_exprs = std::string(str, str + len); - delete[] str; + const auto out_expr_list_a = local_engine::getByteArrayElementsSafe(env, out_expr_list); + const std::string::size_type out_expr_list_size = out_expr_list_a.length(); + out_exprs = std::string{reinterpret_cast(out_expr_list_a.elems()), out_expr_list_size}; } local_engine::SplitOptions options{ @@ -772,14 +742,12 @@ JNIEXPORT jobject Java_org_apache_gluten_vectorized_CHBlockConverterJniWrapper_c DB::Block * block = reinterpret_cast(block_address); if (masks != nullptr) { - jint size = env->GetArrayLength(masks); - jboolean is_cp = JNI_FALSE; - jint * values = env->GetIntArrayElements(masks, &is_cp); + auto safeArray = local_engine::getIntArrayElementsSafe(env, masks); mask = std::make_unique>(); - for (int j = 0; j < size; j++) - mask->push_back(values[j]); - env->ReleaseIntArrayElements(masks, values, JNI_ABORT); + for (int j = 0; j < safeArray.length(); j++) + mask->push_back(safeArray.elems()[j]); } + spark_row_info = converter.convertCHColumnToSparkRow(*block, mask); auto * offsets_arr = env->NewLongArray(spark_row_info->getNumRows()); @@ -925,47 +893,39 @@ JNIEXPORT jlong Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniW LOCAL_ENGINE_JNI_METHOD_START auto query_context = local_engine::getAllocator(allocator_id)->query_context; // by task update new configs ( in case of dynamic config update ) - std::string::size_type conf_plan_buf_size = env->GetArrayLength(conf_plan); - jbyte * conf_plan_buf_addr = env->GetByteArrayElements(conf_plan, nullptr); + const auto conf_plan_a = local_engine::getByteArrayElementsSafe(env, conf_plan); + const std::string::size_type conf_plan_size = conf_plan_a.length(); local_engine::BackendInitializerUtil::updateConfig( - query_context, {reinterpret_cast(conf_plan_buf_addr), conf_plan_buf_size}); + query_context, {reinterpret_cast(conf_plan_a.elems()), conf_plan_size}); const auto uuid_str = jstring2string(env, uuid_); const auto task_id = jstring2string(env, task_id_); const auto partition_dir = jstring2string(env, partition_dir_); const auto bucket_dir = jstring2string(env, bucket_dir_); - jsize plan_buf_size = env->GetArrayLength(plan_); - jbyte * plan_buf_addr = env->GetByteArrayElements(plan_, nullptr); - std::string plan_str; - plan_str.assign(reinterpret_cast(plan_buf_addr), plan_buf_size); + const auto plan_a = local_engine::getByteArrayElementsSafe(env, plan_); - jsize split_info_size = env->GetArrayLength(split_info_); - jbyte * split_info_addr = env->GetByteArrayElements(split_info_, nullptr); - std::string split_info_str; - split_info_str.assign(reinterpret_cast(split_info_addr), split_info_size); - - auto plan_ptr = std::make_unique(); /// https://stackoverflow.com/questions/52028583/getting-error-parsing-protobuf-data /// Parsing may fail when the number of recursive layers is large. /// Here, set a limit large enough to avoid this problem. /// Once this problem occurs, it is difficult to troubleshoot, because the pb of c++ will not provide any valid information - google::protobuf::io::CodedInputStream coded_in(reinterpret_cast(plan_str.data()), static_cast(plan_str.size())); + google::protobuf::io::CodedInputStream coded_in(plan_a.elems(), plan_a.length()); coded_in.SetRecursionLimit(100000); - auto ok = plan_ptr->ParseFromCodedStream(&coded_in); - if (!ok) + substrait::Plan plan_ptr; + if (!plan_ptr.ParseFromCodedStream(&coded_in)) throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from string failed"); + const auto split_info_a = local_engine::getByteArrayElementsSafe(env, split_info_); + const std::string::size_type split_info_size = split_info_a.length(); + std::string split_info_str{reinterpret_cast(split_info_a.elems()), split_info_size}; + substrait::ReadRel::ExtensionTable extension_table = local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); auto merge_tree_table = local_engine::MergeTreeRelParser::parseMergeTreeTable(extension_table); auto uuid = uuid_str + "_" + task_id; auto * writer = new local_engine::SparkMergeTreeWriter(merge_tree_table, query_context, uuid, partition_dir, bucket_dir); - env->ReleaseByteArrayElements(plan_, plan_buf_addr, JNI_ABORT); - env->ReleaseByteArrayElements(split_info_, split_info_addr, JNI_ABORT); - env->ReleaseByteArrayElements(conf_plan, conf_plan_buf_addr, JNI_ABORT); return reinterpret_cast(writer); LOCAL_ENGINE_JNI_METHOD_END(env, 0) } @@ -975,41 +935,32 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn JNIEnv * env, jclass, jbyteArray plan_, jbyteArray read_) { LOCAL_ENGINE_JNI_METHOD_START - jsize plan_buf_size = env->GetArrayLength(plan_); - jbyte * plan_buf_addr = env->GetByteArrayElements(plan_, nullptr); - std::string plan_str; - plan_str.assign(reinterpret_cast(plan_buf_addr), plan_buf_size); + const auto plan_a = local_engine::getByteArrayElementsSafe(env, plan_); + const std::string::size_type plan_size = plan_a.length(); - auto plan_ptr = std::make_unique(); - if (!plan_ptr->ParseFromString(plan_str)) + substrait::Plan plan_ptr; + if (!plan_ptr.ParseFromString({reinterpret_cast(plan_a.elems()), plan_size})) throw Exception(DB::ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from string failed"); - jsize read_buf_size = env->GetArrayLength(read_); - jbyte * read_buf_addr = env->GetByteArrayElements(read_, nullptr); - std::string filter_str; - filter_str.assign(reinterpret_cast(read_buf_addr), read_buf_size); - - auto read_ptr = std::make_unique(); + const auto read_a = local_engine::getByteArrayElementsSafe(env, read_); /// https://stackoverflow.com/questions/52028583/getting-error-parsing-protobuf-data /// Parsing may fail when the number of recursive layers is large. /// Here, set a limit large enough to avoid this problem. /// Once this problem occurs, it is difficult to troubleshoot, because the pb of c++ will not provide any valid information - google::protobuf::io::CodedInputStream coded_in( - reinterpret_cast(filter_str.data()), static_cast(filter_str.size())); + google::protobuf::io::CodedInputStream coded_in(read_a.elems(), read_a.length()); coded_in.SetRecursionLimit(100000); - if (!read_ptr->ParseFromCodedStream(&coded_in)) + substrait::Rel read_ptr; + if (!read_ptr.ParseFromCodedStream(&coded_in)) throw Exception(DB::ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Expression from string failed"); local_engine::SerializedPlanParser parser(local_engine::SerializedPlanParser::global_context); - parser.parseExtensions(plan_ptr->extensions()); + parser.parseExtensions(plan_ptr.extensions()); local_engine::MergeTreeRelParser mergeTreeParser(&parser, local_engine::SerializedPlanParser::global_context); - auto res = mergeTreeParser.filterRangesOnDriver(read_ptr->read()); + auto res = mergeTreeParser.filterRangesOnDriver(read_ptr.read()); - env->ReleaseByteArrayElements(plan_, plan_buf_addr, JNI_ABORT); - env->ReleaseByteArrayElements(read_, read_buf_addr, JNI_ABORT); - return stringTojstring(env, res.c_str()); + return local_engine::charTojstring(env, res.c_str()); LOCAL_ENGINE_JNI_METHOD_END(env, nullptr) } @@ -1052,7 +1003,7 @@ Java_org_apache_spark_sql_execution_datasources_CHDatasourceJniWrapper_closeMerg auto part_infos = writer->getAllPartInfo(); auto json_info = local_engine::SparkMergeTreeWriter::partInfosToJson(part_infos); delete writer; - return stringTojstring(env, json_info.c_str()); + return local_engine::charTojstring(env, json_info.c_str()); LOCAL_ENGINE_JNI_METHOD_END(env, nullptr) } @@ -1077,28 +1028,23 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn const auto partition_dir = jstring2string(env, partition_dir_); const auto bucket_dir = jstring2string(env, bucket_dir_); - jsize plan_buf_size = env->GetArrayLength(plan_); - jbyte * plan_buf_addr = env->GetByteArrayElements(plan_, nullptr); - std::string plan_str; - plan_str.assign(reinterpret_cast(plan_buf_addr), plan_buf_size); - - jsize split_info_size = env->GetArrayLength(split_info_); - jbyte * split_info_addr = env->GetByteArrayElements(split_info_, nullptr); - std::string split_info_str; - split_info_str.assign(reinterpret_cast(split_info_addr), split_info_size); + const auto plan_a = local_engine::getByteArrayElementsSafe(env, plan_); - auto plan_ptr = std::make_unique(); /// https://stackoverflow.com/questions/52028583/getting-error-parsing-protobuf-data /// Parsing may fail when the number of recursive layers is large. /// Here, set a limit large enough to avoid this problem. /// Once this problem occurs, it is difficult to troubleshoot, because the pb of c++ will not provide any valid information - google::protobuf::io::CodedInputStream coded_in(reinterpret_cast(plan_str.data()), static_cast(plan_str.size())); + google::protobuf::io::CodedInputStream coded_in(plan_a.elems(), plan_a.length()); coded_in.SetRecursionLimit(100000); - auto ok = plan_ptr->ParseFromCodedStream(&coded_in); - if (!ok) + substrait::Plan plan_ptr; + if (!plan_ptr.ParseFromCodedStream(&coded_in)) throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_PROTOBUF_SCHEMA, "Parse substrait::Plan from string failed"); + const auto split_info_a = local_engine::getByteArrayElementsSafe(env, split_info_); + const std::string::size_type split_info_size = split_info_a.length(); + std::string split_info_str{reinterpret_cast(split_info_a.elems()), split_info_size}; + substrait::ReadRel::ExtensionTable extension_table = local_engine::SerializedPlanParser::parseExtensionTable(split_info_str); google::protobuf::StringValue table; table.ParseFromString(extension_table.detail().value()); @@ -1130,10 +1076,7 @@ JNIEXPORT jstring Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn auto json_info = local_engine::SparkMergeTreeWriter::partInfosToJson(res); - env->ReleaseByteArrayElements(plan_, plan_buf_addr, JNI_ABORT); - env->ReleaseByteArrayElements(split_info_, split_info_addr, JNI_ABORT); - - return stringTojstring(env, json_info.c_str()); + return local_engine::charTojstring(env, json_info.c_str()); LOCAL_ENGINE_JNI_METHOD_END(env, nullptr) } @@ -1143,18 +1086,15 @@ JNIEXPORT jobject Java_org_apache_spark_sql_execution_datasources_CHDatasourceJn { LOCAL_ENGINE_JNI_METHOD_START auto * block = reinterpret_cast(blockAddress); - int * pIndice = env->GetIntArrayElements(partitionColIndice, nullptr); - int size = env->GetArrayLength(partitionColIndice); + auto safeArray = local_engine::getIntArrayElementsSafe(env, partitionColIndice); std::vector partition_col_indice_vec; - for (int i = 0; i < size; ++i) - partition_col_indice_vec.push_back(pIndice[i]); + for (int i = 0; i < safeArray.length(); ++i) + partition_col_indice_vec.push_back(safeArray.elems()[i]); - env->ReleaseIntArrayElements(partitionColIndice, pIndice, JNI_ABORT); local_engine::BlockStripes bs = local_engine::BlockStripeSplitter::split(*block, partition_col_indice_vec, hasBucket, reserve_partition_columns); - auto * addresses = env->NewLongArray(bs.block_addresses.size()); env->SetLongArrayRegion(addresses, 0, bs.block_addresses.size(), bs.block_addresses.data()); auto * indices = env->NewIntArray(bs.heading_row_indice.size()); @@ -1181,10 +1121,9 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_StorageJoinBuilder_nativeBuild LOCAL_ENGINE_JNI_METHOD_START const auto hash_table_id = jstring2string(env, key); const auto join_key = jstring2string(env, join_key_); - const jsize struct_size = env->GetArrayLength(named_struct); - jbyte * struct_address = env->GetByteArrayElements(named_struct, nullptr); - std::string struct_string; - struct_string.assign(reinterpret_cast(struct_address), struct_size); + const auto named_struct_a = local_engine::getByteArrayElementsSafe(env, named_struct); + const std::string::size_type struct_size = named_struct_a.length(); + std::string struct_string{reinterpret_cast(named_struct_a.elems()), struct_size}; const auto join_type = static_cast(join_type_); const jsize length = env->GetArrayLength(in); local_engine::ReadBufferFromByteArray read_buffer_from_java_array(in, length); @@ -1192,7 +1131,6 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_StorageJoinBuilder_nativeBuild local_engine::configureCompressedReadBuffer(input); const auto * obj = make_wrapper(local_engine::BroadCastJoinBuilder::buildJoin( hash_table_id, input, row_count_, join_key, join_type, has_mixed_join_condition, struct_string)); - env->ReleaseByteArrayElements(named_struct, struct_address, JNI_ABORT); return obj->instance(); LOCAL_ENGINE_JNI_METHOD_END(env, 0) } @@ -1321,12 +1259,11 @@ Java_org_apache_gluten_vectorized_SimpleExpressionEval_createNativeInstance(JNIE local_engine::SerializedPlanParser parser(context); jobject iter = env->NewGlobalRef(input); parser.addInputIter(iter, false); - std::string::size_type plan_size = env->GetArrayLength(plan); - jbyte * plan_address = env->GetByteArrayElements(plan, nullptr); + const auto plan_a = local_engine::getByteArrayElementsSafe(env, plan); + const std::string::size_type plan_size = plan_a.length(); local_engine::LocalExecutor * executor - = parser.createExecutor({reinterpret_cast(plan_address), plan_size}).release(); + = parser.createExecutor({reinterpret_cast(plan_a.elems()), plan_size}).release(); local_engine::LocalExecutor::addExecutor(executor); - env->ReleaseByteArrayElements(plan, plan_address, JNI_ABORT); return reinterpret_cast(executor); LOCAL_ENGINE_JNI_METHOD_END(env, -1) } From 6b6444e57efb61bff3dd6842029e9feb9dfaa8c4 Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Tue, 2 Jul 2024 20:31:07 +0800 Subject: [PATCH 384/402] [VL] Add test for log function (#6211) --- .../gluten/execution/ScalarFunctionsValidateSuite.scala | 6 ++++++ docs/velox-backend-support-progress.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index 3db0f5e79b75..9d0a926e3b0f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -299,6 +299,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + test("Test log function") { + runQueryAndCompare("SELECT log(10, l_orderkey) from lineitem limit 1") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("Test shiftleft function") { val df = runQueryAndCompare("SELECT shiftleft(int_field1, 1) from datatab limit 1") { checkGlutenOperatorMatch[ProjectExecTransformer] diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md index f39bd7016707..f3f1e100b2b2 100644 --- a/docs/velox-backend-support-progress.md +++ b/docs/velox-backend-support-progress.md @@ -233,7 +233,7 @@ Gluten supports 199 functions. (Drag to right to see all data types) | hex | | | S | | | | | | S | | | | | S | | | S | | | | | | | hypot | | | | | | | | | | | | | | | | | | | | | | | | ln | ln | | S | | | S | S | S | S | S | S | | | | | | | | | | | | -| log | ln | | S | | | S | S | S | S | S | S | | | | | | | | | | | | +| log | ln | log | S | | | S | S | S | S | S | S | | | | | | | | | | | | | log10 | log10 | | S | | | S | S | S | S | S | S | | | | | | | | | | | | | log1p | | | | | | | | | | | | | | | | | | | | | | | | log2 | log2 | | S | | | S | S | S | S | S | S | | | | | | | | | | | | From 2dde39c3ec2bccc11f2c30f9f310b28927c67366 Mon Sep 17 00:00:00 2001 From: Cancai Cai <77189278+caicancai@users.noreply.github.com> Date: Wed, 3 Jul 2024 08:31:09 +0800 Subject: [PATCH 385/402] [MINOR] ADD NOTICE (#6277) * [MINOR] ADD NOTICE * Update NOTICE Some apache projects are mentioned in the license also require to be added into the NOTICE file --------- Co-authored-by: Wei-Ting Chen --- NOTICE | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000000..214ea5a87954 --- /dev/null +++ b/NOTICE @@ -0,0 +1,26 @@ +Apache Gluten(incubating) +Copyright 2023-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Apache Spark +Copyright 2014 and onwards The Apache Software Foundation. + +Apache Celeborn +Copyright 2022-2024 The Apache Software Foundation. + +Apache Uniffle (incubating) +Copyright 2022 and onwards The Apache Software Foundation. + +Apache Iceberg +Copyright 2017-2024 The Apache Software Foundation. + +Apache Parquet MR +Copyright 2014-2024 The Apache Software Foundation. + +Apache ORC +Copyright 2013 and onwards The Apache Software Foundation. + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation. From 403be39775d0042d17ac27e3a8a714016be88400 Mon Sep 17 00:00:00 2001 From: BInwei Yang Date: Tue, 2 Jul 2024 18:37:43 -0700 Subject: [PATCH 386/402] [VL] bug fix for S3 read (#6313) bugfix: Velox s3FileSystem use -1 as default file size. Gluten wrongly passed 0 if file size isn't gotten from Spark. --- cpp/velox/compute/VeloxPlanConverter.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/velox/compute/VeloxPlanConverter.cc b/cpp/velox/compute/VeloxPlanConverter.cc index bcd03b110afd..315ff2da67ad 100644 --- a/cpp/velox/compute/VeloxPlanConverter.cc +++ b/cpp/velox/compute/VeloxPlanConverter.cc @@ -81,7 +81,12 @@ std::shared_ptr parseScanSplitInfo( splitInfo->paths.emplace_back(file.uri_file()); splitInfo->starts.emplace_back(file.start()); splitInfo->lengths.emplace_back(file.length()); - facebook::velox::FileProperties fileProps = {file.properties().filesize(), file.properties().modificationtime()}; + + facebook::velox::FileProperties fileProps; + if (file.has_properties()) { + fileProps.fileSize = file.properties().filesize(); + fileProps.modificationTime = file.properties().modificationtime(); + } splitInfo->properties.emplace_back(fileProps); switch (file.file_format_case()) { case SubstraitFileFormatCase::kOrc: From 7b0caf42cd86a257d3592acffcf131e53e275464 Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Wed, 3 Jul 2024 09:48:17 +0800 Subject: [PATCH 387/402] [GLUTEN-6067][CH] Support Spark3.5 with Scala2.13 for CH backend (#6311) Support Spark3.5 with Scala2.13 for CH backend: 1. add a profile for the Scala 2.13 2. Add `toSeq` for all the ArrayBuffer --- backends-clickhouse/pom.xml | 6 +- .../ClickhouseOptimisticTransaction.scala | 2 +- .../org/apache/spark/sql/delta/Snapshot.scala | 24 +++- .../sql/delta/commands/VacuumCommand.scala | 2 +- .../source/DeltaMergeTreeFileFormat.scala | 2 +- .../clickhouse/CHIteratorApi.scala | 4 +- .../apache/gluten/metrics/MetricsUtil.scala | 10 +- .../utils/MergeTreePartsPartitionsUtil.scala | 2 +- .../MergeTreeFileFormatDataWriter.scala | 10 +- ...ckHouseTPCHParquetAQEConcurrentSuite.scala | 3 +- backends-velox/pom.xml | 6 +- gluten-celeborn/clickhouse/pom.xml | 4 +- gluten-core/pom.xml | 6 +- .../org/apache/gluten/GlutenPlugin.scala | 2 +- .../execution/WholeStageTransformer.scala | 2 +- .../gluten/expression/ConverterUtils.scala | 6 +- .../gluten/expression/UDFMappings.scala | 6 +- .../enumerated/EnumeratedApplier.scala | 7 +- .../columnar/heuristic/HeuristicApplier.scala | 7 +- .../columnar/validator/Validators.scala | 2 +- .../gluten/planner/GlutenOptimization.scala | 2 +- .../softaffinity/SoftAffinityManager.scala | 2 +- .../spark/sql/execution/GlutenImplicits.scala | 4 +- .../execution/ShuffledColumnarBatchRDD.scala | 2 +- .../sql/hive/HivePartitionConverter.scala | 7 +- .../gluten/ras/memo/ForwardMemoTable.scala | 2 +- .../org/apache/gluten/ras/path/PathMask.scala | 4 +- .../gluten/ras/mock/MockMemoState.scala | 4 +- gluten-ras/pom.xml | 6 +- .../apache/spark/sql/GlutenTestsTrait.scala | 2 +- gluten-ut/pom.xml | 6 +- .../parquet/GlutenParquetFilterSuite.scala | 1 - .../parquet/GlutenParquetRowIndexSuite.scala | 1 + pom.xml | 112 +++++++++++++++++- shims/common/pom.xml | 2 + shims/pom.xml | 2 +- .../sql/shims/spark34/Spark34Shims.scala | 5 +- shims/spark35/pom.xml | 6 +- .../sql/shims/spark35/Spark35Shims.scala | 7 +- 39 files changed, 223 insertions(+), 67 deletions(-) diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 27ebd75f2d59..5672056b4160 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -100,7 +100,7 @@ org.scalacheck scalacheck_${scala.binary.version} - 1.13.5 + 1.17.0 test @@ -126,13 +126,13 @@ org.scalatestplus - scalatestplus-mockito_2.12 + scalatestplus-mockito_${scala.binary.version} 1.0.0-M2 test org.scalatestplus - scalatestplus-scalacheck_2.12 + scalatestplus-scalacheck_${scala.binary.version} 3.1.0.0-RC2 test diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index d8ab2c1d078c..9e79c4f2e984 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -50,7 +50,7 @@ class ClickhouseOptimisticTransaction( def this( deltaLog: DeltaLog, catalogTable: Option[CatalogTable], - snapshotOpt: Option[Snapshot] = None) { + snapshotOpt: Option[Snapshot] = None) = { this( deltaLog, catalogTable, diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala index 13a91f051dae..8836f7c88d23 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala @@ -33,7 +33,6 @@ import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.delta.util.DeltaCommitFileProvider import org.apache.spark.sql.delta.util.FileNames import org.apache.spark.sql.delta.util.StateCache -import org.apache.spark.sql.util.ScalaExtensions._ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql._ @@ -126,7 +125,27 @@ class Snapshot( * This potentially triggers an IO operation to read the inCommitTimestamp. * This is a lazy val, so repeated calls will not trigger multiple IO operations. */ - protected lazy val getInCommitTimestampOpt: Option[Long] = + protected lazy val getInCommitTimestampOpt: Option[Long] = { + // --- modified start + // This implicit is for scala 2.12, copy from scala 2.13 + implicit class OptionExtCompanion(opt: Option.type) { + /** + * When a given condition is true, evaluates the a argument and returns Some(a). + * When the condition is false, a is not evaluated and None is returned. + */ + def when[A](cond: Boolean)(a: => A): Option[A] = if (cond) Some(a) else None + + /** + * When a given condition is false, evaluates the a argument and returns Some(a). + * When the condition is true, a is not evaluated and None is returned. + */ + def whenNot[A](cond: Boolean)(a: => A): Option[A] = if (!cond) Some(a) else None + + /** Sum up all the `options`, substituting `default` for each `None`. */ + def sum[N: Numeric](default: N)(options: Option[N]*): N = + options.map(_.getOrElse(default)).sum + } + // --- modified end Option.when(DeltaConfigs.IN_COMMIT_TIMESTAMPS_ENABLED.fromMetaData(metadata)) { _reconstructedProtocolMetadataAndICT.inCommitTimestamp .getOrElse { @@ -158,6 +177,7 @@ class Snapshot( } } } + } private[delta] lazy val nonFileActions: Seq[Action] = { diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 987a7c35fa8b..9f455fb27bb1 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -712,7 +712,7 @@ trait VacuumCommandImpl extends DeltaCommand { // This is never going to be a path relative to `basePath` for DVs. None } - case None => None + case _ => None } } } diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala index dc1d1072816a..2f71a0a0ef7b 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala @@ -55,7 +55,7 @@ class DeltaMergeTreeFileFormat(protocol: Protocol, metadata: Metadata) setIndexKeyOption: Option[Seq[String]], primaryKeyOption: Option[Seq[String]], clickhouseTableConfigs: Map[String, String], - partitionColumns: Seq[String]) { + partitionColumns: Seq[String]) = { this(protocol, metadata) this.database = database this.tableName = tableName diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 376e46ebe975..4b9ec739028f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -58,7 +58,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { } dataSchema += newField } - StructType(dataSchema) + StructType(dataSchema.toSeq) } private def createNativeIterator( @@ -114,7 +114,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { if (scan.fileFormat == ReadFileFormat.TextReadFormat) { val names = ConverterUtils.collectAttributeNamesWithoutExprId(scan.outputAttributes()) - localFilesNode.setFileSchema(getFileSchema(scan.getDataSchema, names.asScala)) + localFilesNode.setFileSchema(getFileSchema(scan.getDataSchema, names.asScala.toSeq)) } } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala index 1376dc6a82d1..e1e0f7c11a09 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala @@ -177,10 +177,12 @@ object MetricsUtil extends Logging { /** Get all processors */ def getAllProcessorList(metricData: MetricsData): Seq[MetricsProcessor] = { - metricData.steps.asScala.flatMap( - step => { - step.processors.asScala - }) + metricData.steps.asScala + .flatMap( + step => { + step.processors.asScala + }) + .toSeq } /** Update extra time metric by the processors */ diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/utils/MergeTreePartsPartitionsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/utils/MergeTreePartsPartitionsUtil.scala index 80257c3b5e64..ac6ac959f97c 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/utils/MergeTreePartsPartitionsUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/utils/MergeTreePartsPartitionsUtil.scala @@ -127,7 +127,7 @@ object MergeTreePartsPartitionsUtil extends Logging { sparkSession ) } - partitions + partitions.toSeq } def genInputPartitionSeq( diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala index 3a68ac16df80..712afb3788d1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala @@ -117,10 +117,12 @@ abstract class MergeTreeFileFormatDataWriter( releaseResources() val (taskCommitMessage, taskCommitTime) = Utils.timeTakenMs { // committer.commitTask(taskAttemptContext) - val statuses = returnedMetrics.map( - v => { - v._2 - }) + val statuses = returnedMetrics + .map( + v => { + v._2 + }) + .toSeq new TaskCommitMessage(statuses) } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetAQEConcurrentSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetAQEConcurrentSuite.scala index 34e9658fb419..9f4befbb01a9 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetAQEConcurrentSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHParquetAQEConcurrentSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.types.DoubleType import java.util.concurrent.ForkJoinPool import scala.collection.parallel.ForkJoinTaskSupport +import scala.collection.parallel.immutable.ParVector class GlutenClickHouseTPCHParquetAQEConcurrentSuite extends GlutenClickHouseTPCHAbstractSuite @@ -74,7 +75,7 @@ class GlutenClickHouseTPCHParquetAQEConcurrentSuite test("fix race condition at the global variable of ColumnarOverrideRules::isAdaptiveContext") { - val queries = ((1 to 22) ++ (1 to 22) ++ (1 to 22) ++ (1 to 22)).par + val queries = ParVector((1 to 22) ++ (1 to 22) ++ (1 to 22) ++ (1 to 22): _*) queries.tasksupport = new ForkJoinTaskSupport(new ForkJoinPool(22)) queries.map(queryId => runTPCHQuery(queryId) { df => }) diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index 70b8b901baac..de529a34df2f 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -87,7 +87,7 @@ org.scalacheck scalacheck_${scala.binary.version} - 1.13.5 + 1.17.0 test @@ -113,13 +113,13 @@ org.scalatestplus - scalatestplus-mockito_2.12 + scalatestplus-mockito_${scala.binary.version} 1.0.0-M2 test org.scalatestplus - scalatestplus-scalacheck_2.12 + scalatestplus-scalacheck_${scala.binary.version} 3.1.0.0-RC2 test diff --git a/gluten-celeborn/clickhouse/pom.xml b/gluten-celeborn/clickhouse/pom.xml index 74b81031fb38..f17f5968d351 100755 --- a/gluten-celeborn/clickhouse/pom.xml +++ b/gluten-celeborn/clickhouse/pom.xml @@ -127,7 +127,7 @@ org.scalatestplus - scalatestplus-mockito_2.12 + scalatestplus-mockito_${scala.binary.version} 1.0.0-M2 test @@ -138,7 +138,7 @@ org.scalatestplus - scalatestplus-scalacheck_2.12 + scalatestplus-scalacheck_${scala.binary.version} 3.1.0.0-RC2 test diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml index 740de5928f26..880eddb4ee95 100644 --- a/gluten-core/pom.xml +++ b/gluten-core/pom.xml @@ -84,7 +84,7 @@ org.scalacheck scalacheck_${scala.binary.version} - 1.13.5 + 1.17.0 test @@ -111,13 +111,13 @@ org.scalatestplus - scalatestplus-mockito_2.12 + scalatestplus-mockito_${scala.binary.version} 1.0.0-M2 test org.scalatestplus - scalatestplus-scalacheck_2.12 + scalatestplus-scalacheck_${scala.binary.version} 3.1.0.0-RC2 test diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 0f397c69263c..16929ca4bd4a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -300,7 +300,7 @@ private[gluten] class GlutenSessionExtensions extends (SparkSessionExtensions => } private[gluten] trait GlutenSparkExtensionsInjector { - def inject(extensions: SparkSessionExtensions) + def inject(extensions: SparkSessionExtensions): Unit } private[gluten] object GlutenPlugin { diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala index a49e8aa518b6..78132c08c782 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala @@ -265,7 +265,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f } transformChildren(child, basicScanExecTransformers) - basicScanExecTransformers + basicScanExecTransformers.toSeq } override def doExecuteColumnar(): RDD[ColumnarBatch] = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala index a944de3d3cef..473ee7f9d62f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala @@ -73,7 +73,7 @@ object ConverterUtils extends Logging { } def collectAttributeTypeNodes(attributes: JList[Attribute]): JList[TypeNode] = { - collectAttributeTypeNodes(attributes.asScala) + collectAttributeTypeNodes(attributes.asScala.toSeq) } def collectAttributeTypeNodes(attributes: Seq[Attribute]): JList[TypeNode] = { @@ -85,7 +85,7 @@ object ConverterUtils extends Logging { } def collectAttributeNamesWithExprId(attributes: JList[Attribute]): JList[String] = { - collectAttributeNamesWithExprId(attributes.asScala) + collectAttributeNamesWithExprId(attributes.asScala.toSeq) } def collectAttributeNamesWithExprId(attributes: Seq[Attribute]): JList[String] = { @@ -197,7 +197,7 @@ object ConverterUtils extends Logging { val (field, nullable) = parseFromSubstraitType(typ) StructField("", field, nullable) } - (StructType(fields), isNullable(substraitType.getStruct.getNullability)) + (StructType(fields.toSeq), isNullable(substraitType.getStruct.getNullability)) case Type.KindCase.LIST => val list = substraitType.getList val (elementType, containsNull) = parseFromSubstraitType(list.getType) diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/UDFMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/UDFMappings.scala index 7c836252b92d..3b64c5117fbb 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/UDFMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/UDFMappings.scala @@ -32,7 +32,7 @@ object UDFMappings extends Logging { val pythonUDFMap: Map[String, String] = Map() val scalaUDFMap: Map[String, String] = Map() - private def appendKVToMap(key: String, value: String, res: Map[String, String]) { + private def appendKVToMap(key: String, value: String, res: Map[String, String]): Unit = { if (key.isEmpty || value.isEmpty()) { throw new IllegalArgumentException(s"key:$key or value:$value is empty") } @@ -46,7 +46,7 @@ object UDFMappings extends Logging { res.put(key.toLowerCase(Locale.ROOT), value) } - private def parseStringToMap(input: String, res: Map[String, String]) { + private def parseStringToMap(input: String, res: Map[String, String]): Unit = { input.split(",").map { item => val keyValue = item.split(":") @@ -57,7 +57,7 @@ object UDFMappings extends Logging { } } - def loadFromSparkConf(conf: SparkConf) { + def loadFromSparkConf(conf: SparkConf): Unit = { val strHiveUDFs = conf.get(GlutenConfig.GLUTEN_SUPPORTED_HIVE_UDFS, "") if (!StringUtils.isBlank(strHiveUDFs)) { parseStringToMap(strHiveUDFs, hiveUDFMap) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index d5260f66adba..3d7509abc631 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -45,7 +45,12 @@ class EnumeratedApplier(session: SparkSession) with Logging with LogLevelUtil { // An empirical value. - private val aqeStackTraceIndex = 16 + private val aqeStackTraceIndex = + if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.12"))) { + 16 + } else { + 14 + } private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index d925bc231cd9..34bcf3220daa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -40,7 +40,12 @@ class HeuristicApplier(session: SparkSession) with Logging with LogLevelUtil { // This is an empirical value, may need to be changed for supporting other versions of spark. - private val aqeStackTraceIndex = 19 + private val aqeStackTraceIndex = + if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.12"))) { + 19 + } else { + 17 + } private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex) override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index 56b63ef8457a..2103537500aa 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -97,7 +97,7 @@ object Validators { if (buffer.isEmpty) { NoopValidator } else { - new ValidatorPipeline(buffer) + new ValidatorPipeline(buffer.toSeq) } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala index 555e7d6143bc..5b24f596395d 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/GlutenOptimization.scala @@ -61,7 +61,7 @@ object GlutenOptimization { GlutenMetadataModel(), GlutenPropertyModel(), GlutenExplain, - RasRule.Factory.reuse(rules)) + RasRule.Factory.reuse(rules.toSeq)) } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala index 72d590d04c55..278e1b550092 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala @@ -263,7 +263,7 @@ abstract class AffinityManager extends LogLevelUtil with Logging { rand.shuffle(hosts) logOnLevel(logLevel, s"get host for $f: ${hosts.distinct.mkString(",")}") } - hosts.distinct + hosts.distinct.toSeq } def updatePartitionMap(f: FilePartition, rddId: Int): Unit = { diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala index b0dc3a958e13..eb42f0a88460 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenImplicits.scala @@ -205,8 +205,8 @@ object GlutenImplicits { FallbackSummary( totalNumGlutenNodes, totalNumFallbackNodes, - totalPhysicalPlanDescription, - totalFallbackNodeToReason + totalPhysicalPlanDescription.toSeq, + totalFallbackNodeToReason.toSeq ) } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala index 42db17b6cd6b..0642c3a24760 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ShuffledColumnarBatchRDD.scala @@ -139,7 +139,7 @@ class ShuffledColumnarBatchRDD( } } - override def clearDependencies() { + override def clearDependencies(): Unit = { super.clearDependencies() dependency = null } diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HivePartitionConverter.scala b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HivePartitionConverter.scala index d76eca3d3c49..3a65d6f559ea 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HivePartitionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HivePartitionConverter.scala @@ -80,9 +80,10 @@ class HivePartitionConverter(hadoopConf: Configuration, session: SparkSession) // just like for Apache Spark. val uri = p.getDataLocation.toUri val partValues: Seq[Any] = { - p.getValues.asScala.zip(partitionColTypes).map { - case (value, dataType) => castFromString(value, dataType) - } + p.getValues.asScala + .zip(partitionColTypes) + .map { case (value, dataType) => castFromString(value, dataType) } + .toSeq } val partValuesAsInternalRow = InternalRow.fromSeq(partValues) diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/memo/ForwardMemoTable.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/memo/ForwardMemoTable.scala index dd4033866f86..b99fb280fe5a 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/memo/ForwardMemoTable.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/memo/ForwardMemoTable.scala @@ -155,7 +155,7 @@ class ForwardMemoTable[T <: AnyRef] private (override val ras: Ras[T]) groupBuffer(id) } - override def allClusterKeys(): Seq[RasClusterKey] = clusterKeyBuffer + override def allClusterKeys(): Seq[RasClusterKey] = clusterKeyBuffer.toSeq override def allGroupIds(): Seq[Int] = { val from = -dummyGroupBuffer.size diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/PathMask.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/PathMask.scala index a8caded407b1..c7dd3d2c0718 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/PathMask.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/path/PathMask.scala @@ -96,7 +96,7 @@ object PathMask { return None } - PathMask(buffer) + PathMask(buffer.toSeq) } Some(out) @@ -168,7 +168,7 @@ object PathMask { dfs(0, 0) - PathMask(buffer) + PathMask(buffer.toSeq) } // Return the sub-mask whose root node is the node at the input index diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala index 37d66e2bd703..1c8458af3c61 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/mock/MockMemoState.scala @@ -102,7 +102,7 @@ object MockMemoState { nodeBuffer ++= nodes } - override def nodes(): Seq[CanonicalNode[T]] = nodeBuffer + override def nodes(): Seq[CanonicalNode[T]] = nodeBuffer.toSeq } object MockMutableCluster { @@ -153,7 +153,7 @@ object MockMemoState { group } - def allGroups(): Seq[MockMutableGroup[T]] = groupBuffer + def allGroups(): Seq[MockMutableGroup[T]] = groupBuffer.toSeq } object Factory { diff --git a/gluten-ras/pom.xml b/gluten-ras/pom.xml index e2e8fccb2200..973af760fa87 100644 --- a/gluten-ras/pom.xml +++ b/gluten-ras/pom.xml @@ -32,7 +32,7 @@ org.scalacheck scalacheck_${scala.binary.version} - 1.13.5 + 1.17.0 test @@ -48,13 +48,13 @@ org.scalatestplus - scalatestplus-mockito_2.12 + scalatestplus-mockito_${scala.binary.version} 1.0.0-M2 test org.scalatestplus - scalatestplus-scalacheck_2.12 + scalatestplus-scalacheck_${scala.binary.version} 3.1.0.0-RC2 test diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala index ee765ed36099..5df9d007193e 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala @@ -360,6 +360,6 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait { } _spark.internalCreateDataFrame( _spark.sparkContext.parallelize(Seq(inputRow)), - StructType(structFileSeq)) + StructType(structFileSeq.toSeq)) } } diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml index 79afa94c8e32..1e474042d0fa 100644 --- a/gluten-ut/pom.xml +++ b/gluten-ut/pom.xml @@ -98,7 +98,7 @@ org.scalacheck scalacheck_${scala.binary.version} - 1.13.5 + 1.17.0 test @@ -145,13 +145,13 @@ org.scalatestplus - scalatestplus-mockito_2.12 + scalatestplus-mockito_${scala.binary.version} 1.0.0-M2 test org.scalatestplus - scalatestplus-scalacheck_2.12 + scalatestplus-scalacheck_${scala.binary.version} 3.1.0.0-RC2 test diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala index bb4a78a82795..4141acee3170 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetFilterSuite.scala @@ -37,7 +37,6 @@ import org.apache.spark.util.Utils import org.apache.hadoop.fs.Path import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators} import org.apache.parquet.filter2.predicate.FilterApi._ -import org.apache.parquet.filter2.predicate.Operators import org.apache.parquet.filter2.predicate.Operators.{Column => _, Eq, Gt, GtEq, Lt, LtEq, NotEq} import org.apache.parquet.hadoop.{ParquetFileReader, ParquetInputFormat, ParquetOutputFormat} import org.apache.parquet.hadoop.util.HadoopInputFile diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala index ad41a8395fd0..4c53396792c2 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/GlutenParquetRowIndexSuite.scala @@ -49,6 +49,7 @@ class GlutenParquetRowIndexSuite extends ParquetRowIndexSuite with GlutenSQLTest .getBlocks .asScala .map(_.getRowCount) + .toSeq } private def readRowGroupRowCounts(dir: File): Seq[Seq[Long]] = { diff --git a/pom.xml b/pom.xml index 887839ce5fc0..f12469317747 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,8 @@ 15.0.0-gluten arrow-memory-unsafe 2.7.4 + 2.0.7 + 2.20.0 UTF-8 UTF-8 spark-sql-columnar @@ -113,6 +115,100 @@ + + scala-2.12 + + true + + + + 2.12.15 + 2.12 + + + + scala-2.13 + + 2.13.8 + 2.13 + + + + + + net.alchim31.maven + scala-maven-plugin + + + + -unchecked + -deprecation + -feature + -explaintypes + -target:jvm-1.8 + -Wconf:cat=deprecation:wv,any:e + -Wunused:imports + + -Wconf:cat=scaladoc:wv + -Wconf:cat=lint-multiarg-infix:wv + -Wconf:cat=other-nullary-override:wv + + -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s + -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s + -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s + -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s + -Wconf:msg=method without a parameter list overrides a method with a single empty one:s + + -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e + + -Wconf:cat=unchecked&msg=outer reference:s + -Wconf:cat=unchecked&msg=eliminated by erasure:s + -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s + + + + + + + + + + java-8 @@ -196,6 +292,20 @@ 2.15.1 3.3.4 + + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + org.apache.logging.log4j + log4j-slf4j2-impl + ${log4j.version} + provided + + hadoop-2.7.4 @@ -521,7 +631,7 @@ org.scalatest scalatest_${scala.binary.version} - 3.2.3 + 3.2.16 test diff --git a/shims/common/pom.xml b/shims/common/pom.xml index adf9da7c624e..959a95fc074f 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -61,7 +61,9 @@ -Wconf:cat=deprecation:silent + diff --git a/shims/pom.xml b/shims/pom.xml index 5c17c3ec32cc..61bea7040999 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -37,7 +37,7 @@ org.scalatest scalatest_${scala.binary.version} - 3.2.3 + 3.2.16 test diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala index 420be8511937..203256cf5fec 100644 --- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala +++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala @@ -77,7 +77,10 @@ class Spark34Shims extends SparkShims { Sig[Sec](ExpressionNames.SEC), Sig[Csc](ExpressionNames.CSC), Sig[KnownNullable](KNOWN_NULLABLE), - Sig[Empty2Null](ExpressionNames.EMPTY2NULL) + Sig[Empty2Null](ExpressionNames.EMPTY2NULL), + Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD), + Sig[RoundFloor](ExpressionNames.FLOOR), + Sig[RoundCeil](ExpressionNames.CEIL) ) } diff --git a/shims/spark35/pom.xml b/shims/spark35/pom.xml index 27cd011ac2d6..1c79b882bf4a 100644 --- a/shims/spark35/pom.xml +++ b/shims/spark35/pom.xml @@ -43,13 +43,13 @@ org.apache.spark - spark-catalyst_2.12 + spark-catalyst_${scala.binary.version} provided true org.apache.spark - spark-core_2.12 + spark-core_${scala.binary.version} provided true @@ -109,7 +109,9 @@ -Wconf:cat=deprecation:silent + diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 8ac8d323efd6..821e0f5837d6 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -43,7 +43,7 @@ import org.apache.spark.sql.connector.read.{HasPartitionKey, InputPartition, Sca import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters, ParquetRowIndexUtil} +import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, ParquetFilters} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil @@ -77,7 +77,10 @@ class Spark35Shims extends SparkShims { Sig[Sec](ExpressionNames.SEC), Sig[Csc](ExpressionNames.CSC), Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE), - Sig[Empty2Null](ExpressionNames.EMPTY2NULL) + Sig[Empty2Null](ExpressionNames.EMPTY2NULL), + Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD), + Sig[RoundFloor](ExpressionNames.FLOOR), + Sig[RoundCeil](ExpressionNames.CEIL) ) } From d589aa36862ccc73209d65e03429b4e3bfcbc4c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=98=B3=E9=98=B3?= Date: Wed, 3 Jul 2024 09:59:52 +0800 Subject: [PATCH 388/402] [CORE] Rename TransformHint to FallbackTag (#6254) --- .../clickhouse/CHSparkPlanExecApi.scala | 8 +- .../FallbackBroadcaseHashJoinRules.scala | 16 +-- .../velox/VeloxSparkPlanExecApi.scala | 8 +- .../execution/ScanTransformerFactory.scala | 6 +- .../EnsureLocalSortRequirements.scala | 4 +- .../columnar/ExpandFallbackPolicy.scala | 8 +- ...rmHintRule.scala => FallbackTagRule.scala} | 103 ++++++++---------- .../columnar/OffloadSingleNode.scala | 30 ++--- ...RemoveNativeWriteFilesSortAndProject.scala | 2 +- .../enumerated/EnumeratedApplier.scala | 2 +- .../columnar/heuristic/HeuristicApplier.scala | 4 +- .../RewriteSparkPlanRulesManager.scala | 18 +-- .../columnar/validator/Validators.scala | 6 +- .../execution/GlutenFallbackReporter.scala | 6 +- .../GlutenFormatWriterInjectsBase.scala | 4 +- .../execution/FallbackStrategiesSuite.scala | 6 +- .../execution/FallbackStrategiesSuite.scala | 6 +- .../execution/FallbackStrategiesSuite.scala | 6 +- 18 files changed, 117 insertions(+), 126 deletions(-) rename gluten-core/src/main/scala/org/apache/gluten/extension/columnar/{TransformHintRule.scala => FallbackTagRule.scala} (86%) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index c0dee707ef4f..44aeba021557 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -22,7 +22,7 @@ import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ import org.apache.gluten.extension.{CountDistinctWithoutExpand, FallbackBroadcastHashJoin, FallbackBroadcastHashJoinPrepQueryStage, RewriteToDateExpresstionRule} -import org.apache.gluten.extension.columnar.AddTransformHintRule +import org.apache.gluten.extension.columnar.AddFallbackTagRule import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPreOverrides import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.sql.shims.SparkShimLoader @@ -146,7 +146,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { child match { case scan: FileSourceScanExec if (checkMergeTreeFileFormat(scan.relation)) => - // For the validation phase of the AddTransformHintRule + // For the validation phase of the AddFallbackTagRule CHFilterExecTransformer(condition, child) case scan: FileSourceScanExecTransformerBase if (checkMergeTreeFileFormat(scan.relation)) => // For the transform phase, the FileSourceScanExec is already transformed @@ -226,7 +226,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { // FIXME: The operation happens inside ReplaceSingleNode(). // Caller may not know it adds project on top of the shuffle. val project = TransformPreOverrides().apply( - AddTransformHintRule().apply( + AddFallbackTagRule().apply( ProjectExec(plan.child.output ++ projectExpressions, plan.child))) var newExprs = Seq[Expression]() for (i <- exprs.indices) { @@ -251,7 +251,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { // FIXME: The operation happens inside ReplaceSingleNode(). // Caller may not know it adds project on top of the shuffle. val project = TransformPreOverrides().apply( - AddTransformHintRule().apply( + AddFallbackTagRule().apply( ProjectExec(plan.child.output ++ projectExpressions, plan.child))) var newOrderings = Seq[SortOrder]() for (i <- orderings.indices) { diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcaseHashJoinRules.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcaseHashJoinRules.scala index 873ecb8342a6..59c2d6494bdb 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcaseHashJoinRules.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcaseHashJoinRules.scala @@ -19,7 +19,7 @@ package org.apache.gluten.extension import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ -import org.apache.gluten.extension.columnar.TransformHints.EncodeTransformableTagImplicits +import org.apache.gluten.extension.columnar.FallbackTags.EncodeFallbackTagImplicits import org.apache.gluten.utils.PhysicalPlanSelector import org.apache.spark.sql.SparkSession @@ -61,7 +61,7 @@ case class FallbackBroadcastHashJoinPrepQueryStage(session: SparkSession) extend "columnar broadcast exchange is disabled or " + "columnar broadcast join is disabled") } else { - if (TransformHints.isNotTransformable(bhj)) { + if (FallbackTags.nonEmpty(bhj)) { ValidationResult.notOk("broadcast join is already tagged as not transformable") } else { val bhjTransformer = BackendsApiManager.getSparkPlanExecApiInstance @@ -83,8 +83,8 @@ case class FallbackBroadcastHashJoinPrepQueryStage(session: SparkSession) extend } } } - TransformHints.tagNotTransformable(bhj, isTransformable) - TransformHints.tagNotTransformable(exchange, isTransformable) + FallbackTags.add(bhj, isTransformable) + FallbackTags.add(exchange, isTransformable) case _ => // Skip. This might be the case that the exchange was already // executed in earlier stage @@ -116,7 +116,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl // Currently their doBroadcast() methods just propagate child's broadcast // payloads which is not right in speaking of columnar. if (!enableColumnarBroadcastJoin) { - TransformHints.tagNotTransformable( + FallbackTags.add( bhj, "columnar BroadcastJoin is not enabled in BroadcastHashJoinExec") } else { @@ -149,7 +149,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl case Some(exchange @ BroadcastExchangeExec(mode, child)) => isBhjTransformable.tagOnFallback(bhj) if (!isBhjTransformable.isValid) { - TransformHints.tagNotTransformable(exchange, isBhjTransformable) + FallbackTags.add(exchange, isBhjTransformable) } case None => // we are in AQE, find the hidden exchange @@ -182,7 +182,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl // to conform to the underlying exchange's type, columnar or vanilla exchange match { case BroadcastExchangeExec(mode, child) => - TransformHints.tagNotTransformable( + FallbackTags.add( bhj, "it's a materialized broadcast exchange or reused broadcast exchange") case ColumnarBroadcastExchangeExec(mode, child) => @@ -199,7 +199,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl } } catch { case e: UnsupportedOperationException => - TransformHints.tagNotTransformable( + FallbackTags.add( p, s"${e.getMessage}, original Spark plan is " + s"${p.getClass}(${p.children.toList.map(_.getClass)})") diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 582bf997fba1..e13ebd971ef5 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -25,7 +25,7 @@ import org.apache.gluten.expression._ import org.apache.gluten.expression.ExpressionNames.{TRANSFORM_KEYS, TRANSFORM_VALUES} import org.apache.gluten.expression.aggregate.{HLLAdapter, VeloxBloomFilterAggregate, VeloxCollectList, VeloxCollectSet} import org.apache.gluten.extension._ -import org.apache.gluten.extension.columnar.TransformHints +import org.apache.gluten.extension.columnar.FallbackTags import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.extension.columnar.transition.ConventionFunc.BatchOverride import org.apache.gluten.sql.shims.SparkShimLoader @@ -371,7 +371,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { val newChild = maybeAddAppendBatchesExec(projectTransformer) ColumnarShuffleExchangeExec(shuffle, newChild, newChild.output.drop(1)) } else { - TransformHints.tagNotTransformable(shuffle, validationResult) + FallbackTags.add(shuffle, validationResult) shuffle.withNewChildren(child :: Nil) } case RoundRobinPartitioning(num) if SQLConf.get.sortBeforeRepartition && num > 1 => @@ -397,7 +397,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { projectTransformer } else { val project = ProjectExec(projectList, child) - TransformHints.tagNotTransformable(project, projectBeforeSortValidationResult) + FallbackTags.add(project, projectBeforeSortValidationResult) project } val sortOrder = SortOrder(projectBeforeSort.output.head, Ascending) @@ -410,7 +410,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { val newChild = maybeAddAppendBatchesExec(dropSortColumnTransformer) ColumnarShuffleExchangeExec(shuffle, newChild, newChild.output) } else { - TransformHints.tagNotTransformable(shuffle, validationResult) + FallbackTags.add(shuffle, validationResult) shuffle.withNewChildren(child :: Nil) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala index fcb9e983e76b..44a823834f92 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/ScanTransformerFactory.scala @@ -17,7 +17,7 @@ package org.apache.gluten.execution import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.extension.columnar.TransformHints +import org.apache.gluten.extension.columnar.FallbackTags import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.sql.catalyst.expressions.Expression @@ -99,7 +99,7 @@ object ScanTransformerFactory { transformer } else { val newSource = batchScan.copy(runtimeFilters = transformer.runtimeFilters) - TransformHints.tagNotTransformable(newSource, validationResult.reason.get) + FallbackTags.add(newSource, validationResult.reason.get) newSource } } else { @@ -109,7 +109,7 @@ object ScanTransformerFactory { if (validation) { throw new GlutenNotSupportException(s"Unsupported scan ${batchScan.scan}") } - TransformHints.tagNotTransformable(batchScan, "The scan in BatchScanExec is not supported.") + FallbackTags.add(batchScan, "The scan in BatchScanExec is not supported.") batchScan } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/EnsureLocalSortRequirements.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/EnsureLocalSortRequirements.scala index 0f5fc21aff87..afc29a51e19a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/EnsureLocalSortRequirements.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/EnsureLocalSortRequirements.scala @@ -37,7 +37,7 @@ object EnsureLocalSortRequirements extends Rule[SparkPlan] { requiredOrdering: Seq[SortOrder]): SparkPlan = { val newChild = SortExec(requiredOrdering, global = false, child = originalChild) if (!GlutenConfig.getConf.enableColumnarSort) { - TransformHints.tagNotTransformable(newChild, "columnar Sort is not enabled in SortExec") + FallbackTags.add(newChild, "columnar Sort is not enabled in SortExec") newChild } else { val newChildWithTransformer = @@ -50,7 +50,7 @@ object EnsureLocalSortRequirements extends Rule[SparkPlan] { if (validationResult.isValid) { newChildWithTransformer } else { - TransformHints.tagNotTransformable(newChild, validationResult) + FallbackTags.add(newChild, validationResult) newChild } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala index 4ee153173c5c..e334fcfbce88 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ExpandFallbackPolicy.scala @@ -239,11 +239,11 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP // Propagate fallback reason to vanilla SparkPlan glutenPlan.foreach { case _: GlutenPlan => - case p: SparkPlan if TransformHints.isNotTransformable(p) && p.logicalLink.isDefined => + case p: SparkPlan if FallbackTags.nonEmpty(p) && p.logicalLink.isDefined => originalPlan .find(_.logicalLink.exists(_.fastEquals(p.logicalLink.get))) - .filterNot(TransformHints.isNotTransformable) - .foreach(origin => TransformHints.tag(origin, TransformHints.getHint(p))) + .filterNot(FallbackTags.nonEmpty) + .foreach(origin => FallbackTags.tag(origin, FallbackTags.getTag(p))) case _ => } @@ -278,7 +278,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP ) { plan } else { - TransformHints.tagAllNotTransformable( + FallbackTags.addRecursively( vanillaSparkPlan, TRANSFORM_UNSUPPORTED(fallbackInfo.reason, appendReasonIfExists = false)) FallbackNode(vanillaSparkPlan) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/FallbackTagRule.scala similarity index 86% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/FallbackTagRule.scala index 7fb451057a2e..d34cb0df4e7e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/FallbackTagRule.scala @@ -21,7 +21,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.extension.{GlutenPlan, ValidationResult} -import org.apache.gluten.extension.columnar.TransformHints.EncodeTransformableTagImplicits +import org.apache.gluten.extension.columnar.FallbackTags.EncodeFallbackTagImplicits import org.apache.gluten.extension.columnar.validator.{Validator, Validators} import org.apache.gluten.sql.shims.SparkShimLoader @@ -45,19 +45,19 @@ import org.apache.spark.sql.types.StringType import org.apache.commons.lang3.exception.ExceptionUtils -sealed trait TransformHint { +sealed trait FallbackTag { val stacktrace: Option[String] = - if (TransformHints.DEBUG) { + if (FallbackTags.DEBUG) { Some(ExceptionUtils.getStackTrace(new Throwable())) } else None } case class TRANSFORM_UNSUPPORTED(reason: Option[String], appendReasonIfExists: Boolean = true) - extends TransformHint + extends FallbackTag -object TransformHints { - val TAG: TreeNodeTag[TransformHint] = - TreeNodeTag[TransformHint]("org.apache.gluten.transformhint") +object FallbackTags { + val TAG: TreeNodeTag[FallbackTag] = + TreeNodeTag[FallbackTag]("org.apache.gluten.FallbackTag") val DEBUG = false @@ -69,8 +69,8 @@ object TransformHints { * validation rule. So user should not consider the plan "transformable" unless all validation * rules are passed. */ - def isNotTransformable(plan: SparkPlan): Boolean = { - getHintOption(plan) match { + def nonEmpty(plan: SparkPlan): Boolean = { + getTagOption(plan) match { case Some(TRANSFORM_UNSUPPORTED(_, _)) => true case _ => false } @@ -82,10 +82,10 @@ object TransformHints { * within Gluten transformers. If false, the plan node will be guaranteed fallback to Vanilla plan * node while being implemented. */ - def maybeTransformable(plan: SparkPlan): Boolean = !isNotTransformable(plan) + def maybeOffloadable(plan: SparkPlan): Boolean = !nonEmpty(plan) - def tag(plan: SparkPlan, hint: TransformHint): Unit = { - val mergedHint = getHintOption(plan) + def tag(plan: SparkPlan, hint: FallbackTag): Unit = { + val mergedHint = getTagOption(plan) .map { case originalHint @ TRANSFORM_UNSUPPORTED(Some(originalReason), originAppend) => hint match { @@ -117,33 +117,33 @@ object TransformHints { plan.unsetTagValue(TAG) } - def tagNotTransformable(plan: SparkPlan, validationResult: ValidationResult): Unit = { + def add(plan: SparkPlan, validationResult: ValidationResult): Unit = { if (!validationResult.isValid) { tag(plan, TRANSFORM_UNSUPPORTED(validationResult.reason)) } } - def tagNotTransformable(plan: SparkPlan, reason: String): Unit = { + def add(plan: SparkPlan, reason: String): Unit = { tag(plan, TRANSFORM_UNSUPPORTED(Some(reason))) } - def tagAllNotTransformable(plan: SparkPlan, hint: TRANSFORM_UNSUPPORTED): Unit = { + def addRecursively(plan: SparkPlan, hint: TRANSFORM_UNSUPPORTED): Unit = { plan.foreach { case _: GlutenPlan => // ignore case other => tag(other, hint) } } - def getHint(plan: SparkPlan): TransformHint = { - getHintOption(plan).getOrElse( + def getTag(plan: SparkPlan): FallbackTag = { + getTagOption(plan).getOrElse( throw new IllegalStateException("Transform hint tag not set in plan: " + plan.toString())) } - def getHintOption(plan: SparkPlan): Option[TransformHint] = { + def getTagOption(plan: SparkPlan): Option[FallbackTag] = { plan.getTagValue(TAG) } - implicit class EncodeTransformableTagImplicits(validationResult: ValidationResult) { + implicit class EncodeFallbackTagImplicits(validationResult: ValidationResult) { def tagOnFallback(plan: SparkPlan): Unit = { if (validationResult.isValid) { return @@ -157,7 +157,7 @@ object TransformHints { case class FallbackOnANSIMode(session: SparkSession) extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { if (GlutenConfig.getConf.enableAnsiMode) { - plan.foreach(TransformHints.tagNotTransformable(_, "does not support ansi mode")) + plan.foreach(FallbackTags.add(_, "does not support ansi mode")) } plan } @@ -179,11 +179,11 @@ case class FallbackMultiCodegens(session: SparkSession) extends Rule[SparkPlan] case plan: SortMergeJoinExec if GlutenConfig.getConf.forceShuffledHashJoin => if ((count + 1) >= optimizeLevel) return true plan.children.exists(existsMultiCodegens(_, count + 1)) - case other => false + case _ => false } - def tagNotTransformable(plan: SparkPlan): SparkPlan = { - TransformHints.tagNotTransformable(plan, "fallback multi codegens") + def addFallbackTag(plan: SparkPlan): SparkPlan = { + FallbackTags.add(plan, "fallback multi codegens") plan } @@ -200,35 +200,35 @@ case class FallbackMultiCodegens(session: SparkSession) extends Rule[SparkPlan] } } - def tagNotTransformableRecursive(plan: SparkPlan): SparkPlan = { + def addFallbackTagRecursive(plan: SparkPlan): SparkPlan = { plan match { case p: ShuffleExchangeExec => - tagNotTransformable(p.withNewChildren(p.children.map(tagNotTransformableForMultiCodegens))) + addFallbackTag(p.withNewChildren(p.children.map(tagOnFallbackForMultiCodegens))) case p: BroadcastExchangeExec => - tagNotTransformable(p.withNewChildren(p.children.map(tagNotTransformableForMultiCodegens))) + addFallbackTag(p.withNewChildren(p.children.map(tagOnFallbackForMultiCodegens))) case p: ShuffledHashJoinExec => - tagNotTransformable(p.withNewChildren(p.children.map(tagNotTransformableRecursive))) + addFallbackTag(p.withNewChildren(p.children.map(addFallbackTagRecursive))) case p if !supportCodegen(p) => - p.withNewChildren(p.children.map(tagNotTransformableForMultiCodegens)) + p.withNewChildren(p.children.map(tagOnFallbackForMultiCodegens)) case p if isAQEShuffleReadExec(p) => - p.withNewChildren(p.children.map(tagNotTransformableForMultiCodegens)) + p.withNewChildren(p.children.map(tagOnFallbackForMultiCodegens)) case p: QueryStageExec => p - case p => tagNotTransformable(p.withNewChildren(p.children.map(tagNotTransformableRecursive))) + case p => addFallbackTag(p.withNewChildren(p.children.map(addFallbackTagRecursive))) } } - def tagNotTransformableForMultiCodegens(plan: SparkPlan): SparkPlan = { + def tagOnFallbackForMultiCodegens(plan: SparkPlan): SparkPlan = { plan match { case plan if existsMultiCodegens(plan) => - tagNotTransformableRecursive(plan) + addFallbackTagRecursive(plan) case other => - other.withNewChildren(other.children.map(tagNotTransformableForMultiCodegens)) + other.withNewChildren(other.children.map(tagOnFallbackForMultiCodegens)) } } override def apply(plan: SparkPlan): SparkPlan = { if (physicalJoinOptimize) { - tagNotTransformableForMultiCodegens(plan) + tagOnFallbackForMultiCodegens(plan) } else plan } } @@ -272,13 +272,11 @@ case class FallbackEmptySchemaRelation() extends Rule[SparkPlan] { if (p.children.exists(_.output.isEmpty)) { // Some backends are not eligible to offload plan with zero-column input. // If any child have empty output, mark the plan and that child as UNSUPPORTED. - TransformHints.tagNotTransformable(p, "at least one of its children has empty output") + FallbackTags.add(p, "at least one of its children has empty output") p.children.foreach { child => if (child.output.isEmpty && !child.isInstanceOf[WriteFilesExec]) { - TransformHints.tagNotTransformable( - child, - "at least one of its children has empty output") + FallbackTags.add(child, "at least one of its children has empty output") } } } @@ -291,8 +289,8 @@ case class FallbackEmptySchemaRelation() extends Rule[SparkPlan] { // The doValidate function will be called to check if the conversion is supported. // If false is returned or any unsupported exception is thrown, a row guard will // be added on the top of that plan to prevent actual conversion. -case class AddTransformHintRule() extends Rule[SparkPlan] { - import AddTransformHintRule._ +case class AddFallbackTagRule() extends Rule[SparkPlan] { + import AddFallbackTagRule._ private val glutenConf: GlutenConfig = GlutenConfig.getConf private val validator = Validators .builder() @@ -305,22 +303,15 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { .build() def apply(plan: SparkPlan): SparkPlan = { - addTransformableTags(plan) - } - - /** Inserts a transformable tag on top of those that are not supported. */ - private def addTransformableTags(plan: SparkPlan): SparkPlan = { - // Walk the tree with post-order - val out = plan.mapChildren(addTransformableTags) - addTransformableTag(out) - out + plan.foreachUp { case p => addFallbackTag(p) } + plan } - private def addTransformableTag(plan: SparkPlan): Unit = { + private def addFallbackTag(plan: SparkPlan): Unit = { val outcome = validator.validate(plan) outcome match { case Validator.Failed(reason) => - TransformHints.tagNotTransformable(plan, reason) + FallbackTags.add(plan, reason) return case Validator.Passed => } @@ -508,11 +499,11 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { ) transformer.doValidate().tagOnFallback(plan) case _ => - // Currently we assume a plan to be transformable by default. + // Currently we assume a plan to be offload-able by default. } } catch { case e @ (_: GlutenNotSupportException | _: UnsupportedOperationException) => - TransformHints.tagNotTransformable( + FallbackTags.add( plan, s"${e.getMessage}, original Spark plan is " + s"${plan.getClass}(${plan.children.toList.map(_.getClass)})") @@ -523,7 +514,7 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { } } -object AddTransformHintRule { +object AddFallbackTagRule { implicit private class ValidatorBuilderImplicits(builder: Validators.Builder) { /** @@ -561,9 +552,9 @@ object AddTransformHintRule { } } -case class RemoveTransformHintRule() extends Rule[SparkPlan] { +case class RemoveFallbackTagRule() extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - plan.foreach(TransformHints.untag) + plan.foreach(FallbackTags.untag) plan } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 1f6f840b5552..7a4222b5cb38 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -56,7 +56,7 @@ sealed trait OffloadSingleNode extends Logging { // Aggregation transformation. case class OffloadAggregate() extends OffloadSingleNode with LogLevelUtil { override def offload(plan: SparkPlan): SparkPlan = plan match { - case plan if TransformHints.isNotTransformable(plan) => + case plan if FallbackTags.nonEmpty(plan) => plan case agg: HashAggregateExec => genHashAggregateExec(agg) @@ -72,7 +72,7 @@ case class OffloadAggregate() extends OffloadSingleNode with LogLevelUtil { * the actually used plan for execution. */ private def genHashAggregateExec(plan: HashAggregateExec): SparkPlan = { - if (TransformHints.isNotTransformable(plan)) { + if (FallbackTags.nonEmpty(plan)) { return plan } @@ -92,7 +92,7 @@ case class OffloadAggregate() extends OffloadSingleNode with LogLevelUtil { HashAggregateExecBaseTransformer.from(plan)() case _ => // If the child is not transformable, do not transform the agg. - TransformHints.tagNotTransformable(plan, "child output schema is empty") + FallbackTags.add(plan, "child output schema is empty") plan } } else { @@ -105,7 +105,7 @@ case class OffloadAggregate() extends OffloadSingleNode with LogLevelUtil { // Exchange transformation. case class OffloadExchange() extends OffloadSingleNode with LogLevelUtil { override def offload(plan: SparkPlan): SparkPlan = plan match { - case p if TransformHints.isNotTransformable(p) => + case p if FallbackTags.nonEmpty(p) => p case s: ShuffleExchangeExec if (s.child.supportsColumnar || GlutenConfig.getConf.enablePreferColumnar) && @@ -124,7 +124,7 @@ case class OffloadExchange() extends OffloadSingleNode with LogLevelUtil { case class OffloadJoin() extends OffloadSingleNode with LogLevelUtil { override def offload(plan: SparkPlan): SparkPlan = { - if (TransformHints.isNotTransformable(plan)) { + if (FallbackTags.nonEmpty(plan)) { logDebug(s"Columnar Processing for ${plan.getClass} is under row guard.") return plan } @@ -291,11 +291,11 @@ case class OffloadProject() extends OffloadSingleNode with LogLevelUtil { f } } - val addHint = AddTransformHintRule() + val addHint = AddFallbackTagRule() val newProjectList = projectExec.projectList.filterNot(containsInputFileRelatedExpr) val newProjectExec = ProjectExec(newProjectList, projectExec.child) addHint.apply(newProjectExec) - if (TransformHints.isNotTransformable(newProjectExec)) { + if (FallbackTags.nonEmpty(newProjectExec)) { // Project is still not transformable after remove `input_file_name` expressions. projectExec } else { @@ -305,7 +305,7 @@ case class OffloadProject() extends OffloadSingleNode with LogLevelUtil { // /sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala#L506 val leafScans = findScanNodes(projectExec) assert(leafScans.size <= 1) - if (leafScans.isEmpty || TransformHints.isNotTransformable(leafScans(0))) { + if (leafScans.isEmpty || FallbackTags.nonEmpty(leafScans(0))) { // It means // 1. projectExec has `input_file_name` but no scan child. // 2. It has scan child node but the scan node fallback. @@ -326,12 +326,12 @@ case class OffloadProject() extends OffloadSingleNode with LogLevelUtil { private def genProjectExec(projectExec: ProjectExec): SparkPlan = { if ( - TransformHints.isNotTransformable(projectExec) && + FallbackTags.nonEmpty(projectExec) && BackendsApiManager.getSettings.supportNativeInputFileRelatedExpr() && projectExec.projectList.exists(containsInputFileRelatedExpr) ) { tryOffloadProjectExecWithInputFileRelatedExprs(projectExec) - } else if (TransformHints.isNotTransformable(projectExec)) { + } else if (FallbackTags.nonEmpty(projectExec)) { projectExec } else { logDebug(s"Columnar Processing for ${projectExec.getClass} is currently supported.") @@ -366,7 +366,7 @@ case class OffloadFilter() extends OffloadSingleNode with LogLevelUtil { * the actually used plan for execution. */ private def genFilterExec(filter: FilterExec): SparkPlan = { - if (TransformHints.isNotTransformable(filter)) { + if (FallbackTags.nonEmpty(filter)) { return filter } @@ -375,7 +375,7 @@ case class OffloadFilter() extends OffloadSingleNode with LogLevelUtil { // Push down the left conditions in Filter into FileSourceScan. val newChild: SparkPlan = filter.child match { case scan @ (_: FileSourceScanExec | _: BatchScanExec) => - if (TransformHints.maybeTransformable(scan)) { + if (FallbackTags.maybeOffloadable(scan)) { val newScan = FilterHandler.pushFilterToScan(filter.condition, scan) newScan match { @@ -410,7 +410,7 @@ object OffloadOthers { def doReplace(p: SparkPlan): SparkPlan = { val plan = p - if (TransformHints.isNotTransformable(plan)) { + if (FallbackTags.nonEmpty(plan)) { return plan } plan match { @@ -561,7 +561,7 @@ object OffloadOthers { transformer } else { logDebug(s"Columnar Processing for ${plan.getClass} is currently unsupported.") - TransformHints.tagNotTransformable(plan, validationResult.reason.get) + FallbackTags.add(plan, validationResult.reason.get) plan } case plan: BatchScanExec => @@ -576,7 +576,7 @@ object OffloadOthers { return hiveTableScanExecTransformer } logDebug(s"Columnar Processing for ${plan.getClass} is currently unsupported.") - TransformHints.tagNotTransformable(plan, validateResult.reason.get) + FallbackTags.add(plan, validateResult.reason.get) plan case other => throw new GlutenNotSupportException(s"${other.getClass.toString} is not supported.") diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala index ce94626d999d..d32de32ebb32 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala @@ -76,7 +76,7 @@ object NativeWriteFilesWithSkippingSortAndProject extends Logging { } else { // If we can not transform the project, then we fallback to origin plan which means // we also retain the sort operator. - TransformHints.tagNotTransformable(p, validationResult) + FallbackTags.add(p, validationResult) None } case _ => None diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index 3d7509abc631..519db966c225 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -133,7 +133,7 @@ class EnumeratedApplier(session: SparkSession) // when columnar table cache is enabled. (s: SparkSession) => RemoveGlutenTableCacheColumnarToRow(s), (s: SparkSession) => GlutenFallbackReporter(GlutenConfig.getConf, s), - (_: SparkSession) => RemoveTransformHintRule() + (_: SparkSession) => RemoveFallbackTagRule() ) } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala index 34bcf3220daa..03b2b66b09b3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala @@ -108,7 +108,7 @@ class HeuristicApplier(session: SparkSession) (_: SparkSession) => FallbackEmptySchemaRelation(), (spark: SparkSession) => MergeTwoPhasesHashBaseAggregate(spark), (_: SparkSession) => RewriteSparkPlanRulesManager(), - (_: SparkSession) => AddTransformHintRule() + (_: SparkSession) => AddFallbackTagRule() ) ::: List((_: SparkSession) => TransformPreOverrides()) ::: List( @@ -155,7 +155,7 @@ class HeuristicApplier(session: SparkSession) // when columnar table cache is enabled. (s: SparkSession) => RemoveGlutenTableCacheColumnarToRow(s), (s: SparkSession) => GlutenFallbackReporter(GlutenConfig.getConf, s), - (_: SparkSession) => RemoveTransformHintRule() + (_: SparkSession) => RemoveFallbackTagRule() ) } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala index 34fe34f3f3fa..2abd4d7d4807 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteSparkPlanRulesManager.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar.rewrite -import org.apache.gluten.extension.columnar.{AddTransformHintRule, TransformHint, TransformHints} +import org.apache.gluten.extension.columnar.{AddFallbackTagRule, FallbackTag, FallbackTags} import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.rdd.RDD @@ -49,7 +49,7 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] extends Rule[SparkPlan] { private def mayNeedRewrite(plan: SparkPlan): Boolean = { - TransformHints.maybeTransformable(plan) && { + FallbackTags.maybeOffloadable(plan) && { plan match { case _: SortExec => true case _: TakeOrderedAndProjectExec => true @@ -67,14 +67,14 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] } } - private def getTransformHintBack(rewrittenPlan: SparkPlan): Option[TransformHint] = { + private def getFallbackTagBack(rewrittenPlan: SparkPlan): Option[FallbackTag] = { // The rewritten plan may contain more nodes than origin, for now it should only be // `ProjectExec`. val target = rewrittenPlan.collect { case p if !p.isInstanceOf[ProjectExec] && !p.isInstanceOf[RewrittenNodeWall] => p } assert(target.size == 1) - TransformHints.getHintOption(target.head) + FallbackTags.getTagOption(target.head) } private def applyRewriteRules(origin: SparkPlan): (SparkPlan, Option[String]) = { @@ -93,7 +93,7 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] } override def apply(plan: SparkPlan): SparkPlan = { - val addHint = AddTransformHintRule() + val addHint = AddFallbackTagRule() plan.transformUp { case origin if mayNeedRewrite(origin) => // Add a wall to avoid transforming unnecessary nodes. @@ -104,18 +104,18 @@ class RewriteSparkPlanRulesManager private (rewriteRules: Seq[RewriteSingleNode] // Note, it is not expected, but it happens in CH backend when pulling out // aggregate. // TODO: Fix the exception and remove this branch - TransformHints.tagNotTransformable(origin, error.get) + FallbackTags.add(origin, error.get) origin } else if (withWall.fastEquals(rewrittenPlan)) { // Return origin if the rewrite rules do nothing. - // We do not add tag and leave it to the outside `AddTransformHintRule`. + // We do not add tag and leave it to the outside `AddFallbackTagRule`. origin } else { addHint.apply(rewrittenPlan) - val hint = getTransformHintBack(rewrittenPlan) + val hint = getFallbackTagBack(rewrittenPlan) if (hint.isDefined) { // If the rewritten plan is still not transformable, return the original plan. - TransformHints.tag(origin, hint.get) + FallbackTags.tag(origin, hint.get) origin } else { rewrittenPlan.transformUp { diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index 2103537500aa..959bf808aba4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -19,7 +19,7 @@ package org.apache.gluten.extension.columnar.validator import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.{BackendsApiManager, BackendSettingsApi} import org.apache.gluten.expression.ExpressionUtils -import org.apache.gluten.extension.columnar.{TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackTags, TRANSFORM_UNSUPPORTED} import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.sql.execution._ @@ -108,8 +108,8 @@ object Validators { private object FallbackByHint extends Validator { override def validate(plan: SparkPlan): Validator.OutCome = { - if (TransformHints.isNotTransformable(plan)) { - val hint = TransformHints.getHint(plan).asInstanceOf[TRANSFORM_UNSUPPORTED] + if (FallbackTags.nonEmpty(plan)) { + val hint = FallbackTags.getTag(plan).asInstanceOf[TRANSFORM_UNSUPPORTED] return fail(hint.reason.getOrElse("Reason not recorded")) } pass() diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala index 721a30eb4f40..d41dce882602 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.events.GlutenPlanFallbackEvent import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackTags, TRANSFORM_UNSUPPORTED} import org.apache.gluten.utils.LogLevelUtil import org.apache.spark.sql.SparkSession @@ -57,8 +57,8 @@ case class GlutenFallbackReporter(glutenConfig: GlutenConfig, spark: SparkSessio val validationLogLevel = glutenConfig.validationLogLevel plan.foreachUp { case _: GlutenPlan => // ignore - case p: SparkPlan if TransformHints.isNotTransformable(p) => - TransformHints.getHint(p) match { + case p: SparkPlan if FallbackTags.nonEmpty(p) => + FallbackTags.getTag(p) match { case TRANSFORM_UNSUPPORTED(Some(reason), append) => logFallbackReason(validationLogLevel, p.nodeName, reason) // With in next round stage in AQE, the physical plan would be a new instance that diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala index fbdbeadba886..450b88163afc 100644 --- a/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala +++ b/gluten-core/src/main/scala/org/apache/spark/sql/execution/datasources/GlutenFormatWriterInjectsBase.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.execution.datasources import org.apache.gluten.execution.{ProjectExecTransformer, SortExecTransformer, TransformSupport, WholeStageTransformer} import org.apache.gluten.execution.datasource.GlutenFormatWriterInjects -import org.apache.gluten.extension.columnar.AddTransformHintRule +import org.apache.gluten.extension.columnar.AddFallbackTagRule import org.apache.gluten.extension.columnar.MiscColumnarRules.TransformPreOverrides import org.apache.gluten.extension.columnar.rewrite.RewriteSparkPlanRulesManager @@ -47,7 +47,7 @@ trait GlutenFormatWriterInjectsBase extends GlutenFormatWriterInjects { val rules = List( RewriteSparkPlanRulesManager(), - AddTransformHintRule(), + AddFallbackTagRule(), TransformPreOverrides() ) val transformed = rules.foldLeft(plan) { case (latestPlan, rule) => rule.apply(latestPlan) } diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index fff883d49e86..b9c9d8a270bf 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, FallbackTags, TRANSFORM_UNSUPPORTED} import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector @@ -124,10 +124,10 @@ class FallbackStrategiesSuite extends GlutenSQLTestsTrait { testGluten("Tag not transformable more than once") { val originalPlan = UnaryOp1(LeafOp(supportsColumnar = true)) - TransformHints.tag(originalPlan, TRANSFORM_UNSUPPORTED(Some("fake reason"))) + FallbackTags.tag(originalPlan, TRANSFORM_UNSUPPORTED(Some("fake reason"))) val rule = FallbackEmptySchemaRelation() val newPlan = rule.apply(originalPlan) - val reason = TransformHints.getHint(newPlan).asInstanceOf[TRANSFORM_UNSUPPORTED].reason + val reason = FallbackTags.getTag(newPlan).asInstanceOf[TRANSFORM_UNSUPPORTED].reason assert(reason.isDefined) if (BackendsApiManager.getSettings.fallbackOnEmptySchema(newPlan)) { assert( diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 7976288dd4ef..8ce0af8df051 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, FallbackTags, TRANSFORM_UNSUPPORTED} import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector @@ -125,10 +125,10 @@ class FallbackStrategiesSuite extends GlutenSQLTestsTrait { testGluten("Tag not transformable more than once") { val originalPlan = UnaryOp1(LeafOp(supportsColumnar = true)) - TransformHints.tag(originalPlan, TRANSFORM_UNSUPPORTED(Some("fake reason"))) + FallbackTags.tag(originalPlan, TRANSFORM_UNSUPPORTED(Some("fake reason"))) val rule = FallbackEmptySchemaRelation() val newPlan = rule.apply(originalPlan) - val reason = TransformHints.getHint(newPlan).asInstanceOf[TRANSFORM_UNSUPPORTED].reason + val reason = FallbackTags.getTag(newPlan).asInstanceOf[TRANSFORM_UNSUPPORTED].reason assert(reason.isDefined) if (BackendsApiManager.getSettings.fallbackOnEmptySchema(newPlan)) { assert( diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 7976288dd4ef..8ce0af8df051 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.execution.BasicScanExecTransformer import org.apache.gluten.extension.GlutenPlan -import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, TRANSFORM_UNSUPPORTED, TransformHints} +import org.apache.gluten.extension.columnar.{FallbackEmptySchemaRelation, FallbackTags, TRANSFORM_UNSUPPORTED} import org.apache.gluten.extension.columnar.heuristic.HeuristicApplier import org.apache.gluten.extension.columnar.transition.InsertTransitions import org.apache.gluten.utils.QueryPlanSelector @@ -125,10 +125,10 @@ class FallbackStrategiesSuite extends GlutenSQLTestsTrait { testGluten("Tag not transformable more than once") { val originalPlan = UnaryOp1(LeafOp(supportsColumnar = true)) - TransformHints.tag(originalPlan, TRANSFORM_UNSUPPORTED(Some("fake reason"))) + FallbackTags.tag(originalPlan, TRANSFORM_UNSUPPORTED(Some("fake reason"))) val rule = FallbackEmptySchemaRelation() val newPlan = rule.apply(originalPlan) - val reason = TransformHints.getHint(newPlan).asInstanceOf[TRANSFORM_UNSUPPORTED].reason + val reason = FallbackTags.getTag(newPlan).asInstanceOf[TRANSFORM_UNSUPPORTED].reason assert(reason.isDefined) if (BackendsApiManager.getSettings.fallbackOnEmptySchema(newPlan)) { assert( From 47fa44f1d443917baf8849c3518f504956d29b52 Mon Sep 17 00:00:00 2001 From: LiuNeng <1398775315@qq.com> Date: Wed, 3 Jul 2024 10:14:05 +0800 Subject: [PATCH 389/402] [CH] Support replicaterows #6308 What changes were proposed in this pull request? Support expression ReplicateRows in spark How was this patch tested? unit tests --- .../GlutenClickhouseFunctionSuite.scala | 15 +++ .../Operator/ReplicateRowsStep.cpp | 108 ++++++++++++++++++ .../local-engine/Operator/ReplicateRowsStep.h | 48 ++++++++ .../local-engine/Parser/ProjectRelParser.cpp | 34 ++++++ cpp-ch/local-engine/Parser/ProjectRelParser.h | 4 +- cpp-ch/local-engine/Parser/RelParser.h | 1 - .../Parser/SerializedPlanParser.cpp | 6 + .../Parser/SerializedPlanParser.h | 3 + .../expression/ExpressionMappings.scala | 1 + .../gluten/expression/ExpressionNames.scala | 1 + 10 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 cpp-ch/local-engine/Operator/ReplicateRowsStep.cpp create mode 100644 cpp-ch/local-engine/Operator/ReplicateRowsStep.h diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala index 26e997281221..8853dfc77853 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala @@ -211,4 +211,19 @@ class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite { compareResultsAgainstVanillaSpark(query_sql, true, { _ => }) spark.sql("drop table test") } + + test("intersect all") { + spark.sql("create table t1 (a int, b string) using parquet") + spark.sql("insert into t1 values (1, '1'),(2, '2'),(3, '3'),(4, '4'),(5, '5'),(6, '6')") + spark.sql("create table t2 (a int, b string) using parquet") + spark.sql("insert into t2 values (4, '4'),(5, '5'),(6, '6'),(7, '7'),(8, '8'),(9, '9')") + runQueryAndCompare( + """ + |SELECT a,b FROM t1 INTERSECT ALL SELECT a,b FROM t2 + |""".stripMargin + )(df => checkFallbackOperators(df, 0)) + spark.sql("drop table t1") + spark.sql("drop table t2") + } + } diff --git a/cpp-ch/local-engine/Operator/ReplicateRowsStep.cpp b/cpp-ch/local-engine/Operator/ReplicateRowsStep.cpp new file mode 100644 index 000000000000..f2d4bc8a865d --- /dev/null +++ b/cpp-ch/local-engine/Operator/ReplicateRowsStep.cpp @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ReplicateRowsStep.h" + +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +} +} + +namespace local_engine +{ +static DB::ITransformingStep::Traits getTraits() +{ + return DB::ITransformingStep::Traits + { + { + .preserves_number_of_streams = true, + .preserves_sorting = false, + }, + { + .preserves_number_of_rows = false, + } + }; +} + +ReplicateRowsStep::ReplicateRowsStep(const DB::DataStream & input_stream) + : ITransformingStep(input_stream, transformHeader(input_stream.header), getTraits()) +{ +} + +DB::Block ReplicateRowsStep::transformHeader(const DB::Block& input) +{ + DB::Block output; + for (int i = 1; i < input.columns(); i++) + { + output.insert(input.getByPosition(i)); + } + return output; +} + +void ReplicateRowsStep::transformPipeline( + DB::QueryPipelineBuilder & pipeline, + const DB::BuildQueryPipelineSettings & /*settings*/) +{ + pipeline.addSimpleTransform( + [&](const DB::Block & header) + { + return std::make_shared(header); + }); +} + +void ReplicateRowsStep::updateOutputStream() +{ + output_stream = createOutputStream(input_streams.front(), transformHeader(input_streams.front().header), getDataStreamTraits()); +} + +ReplicateRowsTransform::ReplicateRowsTransform(const DB::Block & input_header_) + : ISimpleTransform(input_header_, ReplicateRowsStep::transformHeader(input_header_), true) +{ +} + +void ReplicateRowsTransform::transform(DB::Chunk & chunk) +{ + auto replica_column = chunk.getColumns().front(); + size_t total_rows = 0; + for (int i = 0; i < replica_column->size(); i++) + { + total_rows += replica_column->get64(i); + } + + auto columns = chunk.detachColumns(); + DB::MutableColumns mutable_columns; + for (int i = 1; i < columns.size(); i++) + { + mutable_columns.push_back(columns[i]->cloneEmpty()); + mutable_columns.back()->reserve(total_rows); + DB::ColumnPtr src_col = columns[i]; + DB::MutableColumnPtr & cur = mutable_columns.back(); + for (int j = 0; j < replica_column->size(); j++) + { + cur->insertManyFrom(*src_col, j, replica_column->getUInt(j)); + } + } + + chunk.setColumns(std::move(mutable_columns), total_rows); +} +} \ No newline at end of file diff --git a/cpp-ch/local-engine/Operator/ReplicateRowsStep.h b/cpp-ch/local-engine/Operator/ReplicateRowsStep.h new file mode 100644 index 000000000000..f588bf0ceb8c --- /dev/null +++ b/cpp-ch/local-engine/Operator/ReplicateRowsStep.h @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +namespace local_engine +{ + +class ReplicateRowsStep : public DB::ITransformingStep +{ +public: + ReplicateRowsStep(const DB::DataStream& input_stream); + + static DB::Block transformHeader(const DB::Block& input); + + String getName() const override { return "ReplicateRowsStep"; } + void transformPipeline(DB::QueryPipelineBuilder& pipeline, + const DB::BuildQueryPipelineSettings& settings) override; +private: + void updateOutputStream() override; +}; + +class ReplicateRowsTransform : public DB::ISimpleTransform +{ +public: + ReplicateRowsTransform(const DB::Block& input_header_); + + String getName() const override { return "ReplicateRowsTransform"; } + void transform(DB::Chunk&) override; + +}; +} diff --git a/cpp-ch/local-engine/Parser/ProjectRelParser.cpp b/cpp-ch/local-engine/Parser/ProjectRelParser.cpp index eb190101f170..2f75ac396dfe 100644 --- a/cpp-ch/local-engine/Parser/ProjectRelParser.cpp +++ b/cpp-ch/local-engine/Parser/ProjectRelParser.cpp @@ -21,6 +21,9 @@ #include #include #include +#include + +using namespace DB; namespace local_engine { @@ -109,15 +112,46 @@ ProjectRelParser::SplittedActionsDAGs ProjectRelParser::splitActionsDAGInGenerat return res; } +bool ProjectRelParser::isReplicateRows(substrait::GenerateRel rel) +{ + return plan_parser->isFunction(rel.generator().scalar_function(), "replicaterows"); +} + +DB::QueryPlanPtr ProjectRelParser::parseReplicateRows(DB::QueryPlanPtr query_plan, substrait::GenerateRel generate_rel) +{ + std::vector expressions; + for (int i = 0; i < generate_rel.generator().scalar_function().arguments_size(); ++i) + { + expressions.emplace_back(generate_rel.generator().scalar_function().arguments(i).value()); + } + auto header = query_plan->getCurrentDataStream().header; + auto actions_dag = expressionsToActionsDAG(expressions, header); + auto before_replicate_rows = std::make_unique(query_plan->getCurrentDataStream(), actions_dag); + before_replicate_rows->setStepDescription("Before ReplicateRows"); + steps.emplace_back(before_replicate_rows.get()); + query_plan->addStep(std::move(before_replicate_rows)); + + auto replicate_rows_step = std::make_unique(query_plan->getCurrentDataStream()); + replicate_rows_step->setStepDescription("ReplicateRows"); + steps.emplace_back(replicate_rows_step.get()); + query_plan->addStep(std::move(replicate_rows_step)); + return query_plan; +} + DB::QueryPlanPtr ProjectRelParser::parseGenerate(DB::QueryPlanPtr query_plan, const substrait::Rel & rel, std::list & /*rel_stack_*/) { const auto & generate_rel = rel.generate(); + if (isReplicateRows(generate_rel)) + { + return parseReplicateRows(std::move(query_plan), generate_rel); + } std::vector expressions; for (int i = 0; i < generate_rel.child_output_size(); ++i) { expressions.emplace_back(generate_rel.child_output(i)); } + expressions.emplace_back(generate_rel.generator()); auto header = query_plan->getCurrentDataStream().header; auto actions_dag = expressionsToActionsDAG(expressions, header); diff --git a/cpp-ch/local-engine/Parser/ProjectRelParser.h b/cpp-ch/local-engine/Parser/ProjectRelParser.h index ae5693914475..48a16d774d88 100644 --- a/cpp-ch/local-engine/Parser/ProjectRelParser.h +++ b/cpp-ch/local-engine/Parser/ProjectRelParser.h @@ -19,7 +19,6 @@ #include #include #include -#include namespace local_engine { @@ -50,6 +49,9 @@ class ProjectRelParser : public RelParser /// Split actions_dag of generate rel into 3 parts: before array join + during array join + after array join static SplittedActionsDAGs splitActionsDAGInGenerate(ActionsDAGPtr actions_dag); + bool isReplicateRows(substrait::GenerateRel rel); + + DB::QueryPlanPtr parseReplicateRows(QueryPlanPtr query_plan, substrait::GenerateRel generate_rel); const substrait::Rel & getSingleInput(const substrait::Rel & rel) override { diff --git a/cpp-ch/local-engine/Parser/RelParser.h b/cpp-ch/local-engine/Parser/RelParser.h index 6ca8af535955..0228c2867a26 100644 --- a/cpp-ch/local-engine/Parser/RelParser.h +++ b/cpp-ch/local-engine/Parser/RelParser.h @@ -85,7 +85,6 @@ class RelParser static std::map parseFormattedRelAdvancedOptimization(const substrait::extensions::AdvancedExtension &advanced_extension); static std::string getStringConfig(const std::map & configs, const std::string & key, const std::string & default_value = ""); -private: SerializedPlanParser * plan_parser; }; diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 1ee485346d07..8fd573593eb5 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -1162,6 +1162,12 @@ std::pair SerializedPlanParser::convertStructFieldType(const #undef UINT_CONVERT } +bool SerializedPlanParser::isFunction(substrait::Expression_ScalarFunction rel, String function_name) +{ + auto func_signature = function_mapping[std::to_string(rel.function_reference())]; + return func_signature.starts_with(function_name + ":"); +} + ActionsDAGPtr SerializedPlanParser::parseFunction( const Block & header, const substrait::Expression & rel, std::string & result_name, ActionsDAGPtr actions_dag, bool keep_result) { diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index c62dc73c9394..005a2f4b3458 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -255,6 +255,7 @@ class SerializedPlanParser friend class NonNullableColumnsResolver; friend class JoinRelParser; friend class MergeTreeRelParser; + friend class ProjectRelParser; std::unique_ptr createExecutor(DB::QueryPlanPtr query_plan); @@ -391,6 +392,8 @@ class SerializedPlanParser const std::vector & columns, ActionsDAGPtr actions_dag, std::map & nullable_measure_names); static std::pair convertStructFieldType(const DB::DataTypePtr & type, const DB::Field & field); + bool isFunction(substrait::Expression_ScalarFunction rel, String function_name); + int name_no = 0; std::unordered_map function_mapping; std::vector input_iters; diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index 806ec844de60..e7e9c7ffe900 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -282,6 +282,7 @@ object ExpressionMappings { Sig[MonotonicallyIncreasingID](MONOTONICALLY_INCREASING_ID), Sig[SparkPartitionID](SPARK_PARTITION_ID), Sig[WidthBucket](WIDTH_BUCKET), + Sig[ReplicateRows](REPLICATE_ROWS), // Decimal Sig[UnscaledValue](UNSCALED_VALUE), // Generator function diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 7060e297ea10..278f11922645 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -314,6 +314,7 @@ object ExpressionNames { final val SPARK_PARTITION_ID = "spark_partition_id" final val MONOTONICALLY_INCREASING_ID = "monotonically_increasing_id" final val WIDTH_BUCKET = "width_bucket" + final val REPLICATE_ROWS = "replicaterows" // Directly use child expression transformer final val KNOWN_NULLABLE = "known_nullable" From 80bb848eb8bb567abc880f08ca672d0c04b04eb6 Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Wed, 3 Jul 2024 01:27:52 -0500 Subject: [PATCH 390/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240703) (#6314) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240703) * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/64818 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp | 4 +--- cpp-ch/local-engine/Parser/JoinRelParser.cpp | 2 +- cpp-ch/local-engine/tests/benchmark_local_engine.cpp | 2 +- cpp-ch/local-engine/tests/gtest_ch_join.cpp | 2 +- cpp-ch/local-engine/tests/gtest_parser.cpp | 2 ++ 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 54d0a74c5bb4..1630f5760187 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,4 +1,4 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240621 -CH_COMMIT=c811cbb985f +CH_BRANCH=rebase_ch/20240703 +CH_COMMIT=aa71be074ad diff --git a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp index af306564a4c5..326e11a84f81 100644 --- a/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp +++ b/cpp-ch/local-engine/Join/StorageJoinFromReadBuffer.cpp @@ -15,17 +15,15 @@ * limitations under the License. */ #include "StorageJoinFromReadBuffer.h" -#include #include #include -#include +#include #include #include #include #include -#include #include namespace DB diff --git a/cpp-ch/local-engine/Parser/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/JoinRelParser.cpp index 9a3cc91baaa9..a6a146954d6f 100644 --- a/cpp-ch/local-engine/Parser/JoinRelParser.cpp +++ b/cpp-ch/local-engine/Parser/JoinRelParser.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp index 208a3b518d45..43cdab8a41fa 100644 --- a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp +++ b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/cpp-ch/local-engine/tests/gtest_ch_join.cpp b/cpp-ch/local-engine/tests/gtest_ch_join.cpp index 739390302b46..43bac7a59728 100644 --- a/cpp-ch/local-engine/tests/gtest_ch_join.cpp +++ b/cpp-ch/local-engine/tests/gtest_ch_join.cpp @@ -31,7 +31,7 @@ #include #include -#include +#include #include #include diff --git a/cpp-ch/local-engine/tests/gtest_parser.cpp b/cpp-ch/local-engine/tests/gtest_parser.cpp index 485740191ea3..24c796358f45 100644 --- a/cpp-ch/local-engine/tests/gtest_parser.cpp +++ b/cpp-ch/local-engine/tests/gtest_parser.cpp @@ -138,6 +138,7 @@ TEST(LocalExecutor, StorageFileSink) metadata.setColumns(ColumnsDescription::fromNamesAndTypes({{"name", STRING()}, {"value", UINT()}})); StorageMetadataPtr metadata_ptr = std::make_shared(metadata); +/* auto sink = createFilelinkSink( metadata_ptr, "test_table", @@ -150,4 +151,5 @@ TEST(LocalExecutor, StorageFileSink) sink->consume(testChunk()); sink->onFinish(); + */ } \ No newline at end of file From 5af035948e502e06368bc9af39639e0f2fbad654 Mon Sep 17 00:00:00 2001 From: Chang chen Date: Wed, 3 Jul 2024 17:12:31 +0800 Subject: [PATCH 391/402] Revert "[GLUTEN-6122] Fix crash when driver send shutdown command to executor #6130" (#6273) This reverts commit eee234e398c9418b6f5f93dcfb142e0e0948711f. --- cpp-ch/local-engine/Common/CHUtil.cpp | 7 +-- .../Parser/SerializedPlanParser.cpp | 56 +------------------ .../Parser/SerializedPlanParser.h | 12 ---- cpp-ch/local-engine/local_engine_jni.cpp | 9 +-- 4 files changed, 5 insertions(+), 79 deletions(-) diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index 4a21dbe39834..770fbbc59c80 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -822,7 +822,7 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) size_t index_uncompressed_cache_size = config->getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); double index_uncompressed_cache_size_ratio = config->getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio); - + String index_mark_cache_policy = config->getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); size_t index_mark_cache_size = config->getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); double index_mark_cache_size_ratio = config->getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); @@ -986,10 +986,7 @@ void BackendInitializerUtil::updateConfig(const DB::ContextMutablePtr & context, void BackendFinalizerUtil::finalizeGlobally() { - /// Make sure that all active LocalExecutor stop before spark executor shutdown, otherwise crash map happen. - LocalExecutor::cancelAll(); - - /// Make sure client caches release before ClientCacheRegistry + // Make sure client caches release before ClientCacheRegistry ReadBufferBuilderFactory::instance().clean(); StorageMergeTreeFactory::clear(); auto & global_context = SerializedPlanParser::global_context; diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index 8fd573593eb5..8c60c6e500a9 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -2033,33 +2033,6 @@ void SerializedPlanParser::wrapNullable( SharedContextHolder SerializedPlanParser::shared_context; -std::unordered_map LocalExecutor::executors; -std::mutex LocalExecutor::executors_mutex; - -void LocalExecutor::cancelAll() -{ - std::lock_guard lock{executors_mutex}; - - for (auto & [handle, executor] : executors) - executor->asyncCancel(); - - for (auto & [handle, executor] : executors) - executor->waitCancelFinished(); -} - -void LocalExecutor::addExecutor(LocalExecutor * executor) -{ - std::lock_guard lock{executors_mutex}; - Int64 handle = reinterpret_cast(executor); - executors.emplace(handle, executor); -} - -void LocalExecutor::removeExecutor(Int64 handle) -{ - std::lock_guard lock{executors_mutex}; - executors.erase(handle); -} - LocalExecutor::~LocalExecutor() { if (context->getConfigRef().getBool("dump_pipeline", false)) @@ -2127,35 +2100,8 @@ Block * LocalExecutor::nextColumnar() void LocalExecutor::cancel() { - asyncCancel(); - waitCancelFinished(); -} - -void LocalExecutor::asyncCancel() -{ - if (executor && !is_cancelled) - { - LOG_INFO(&Poco::Logger::get("LocalExecutor"), "Cancel LocalExecutor {}", reinterpret_cast(this)); + if (executor) executor->cancel(); - } -} - -void LocalExecutor::waitCancelFinished() -{ - if (executor && !is_cancelled) - { - Stopwatch watch; - Chunk chunk; - while (executor->pull(chunk)) - ; - is_cancelled = true; - - LOG_INFO( - &Poco::Logger::get("LocalExecutor"), - "Finish cancel LocalExecutor {}, takes {} ms", - reinterpret_cast(this), - watch.elapsedMilliseconds()); - } } Block & LocalExecutor::getHeader() diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.h b/cpp-ch/local-engine/Parser/SerializedPlanParser.h index 005a2f4b3458..90086ea28649 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.h +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.h @@ -439,16 +439,9 @@ class LocalExecutor : public BlockIterator void setMetric(RelMetricPtr metric_) { metric = metric_; } void setExtraPlanHolder(std::vector & extra_plan_holder_) { extra_plan_holder = std::move(extra_plan_holder_); } - static void cancelAll(); - static void addExecutor(LocalExecutor * executor); - static void removeExecutor(Int64 handle); - private: std::unique_ptr writeBlockToSparkRow(const DB::Block & block) const; - void asyncCancel(); - void waitCancelFinished(); - /// Dump processor runtime information to log std::string dumpPipeline() const; @@ -461,11 +454,6 @@ class LocalExecutor : public BlockIterator QueryPlanPtr current_query_plan; RelMetricPtr metric; std::vector extra_plan_holder; - std::atomic is_cancelled{false}; - - /// Record all active LocalExecutor in current executor to cancel them when executor receives shutdown command from driver. - static std::unordered_map executors; - static std::mutex executors_mutex; }; diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index 2338bfe8b1e6..695fc8585538 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -259,7 +259,6 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_ const std::string::size_type plan_size = plan_a.length(); local_engine::LocalExecutor * executor = parser.createExecutor({reinterpret_cast(plan_a.elems()), plan_size}).release(); - local_engine::LocalExecutor::addExecutor(executor); LOG_INFO(&Poco::Logger::get("jni"), "Construct LocalExecutor {}", reinterpret_cast(executor)); executor->setMetric(parser.getMetric()); executor->setExtraPlanHolder(parser.extra_plan_holder); @@ -289,17 +288,15 @@ JNIEXPORT jlong Java_org_apache_gluten_vectorized_BatchIterator_nativeCHNext(JNI JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeCancel(JNIEnv * env, jobject /*obj*/, jlong executor_address) { LOCAL_ENGINE_JNI_METHOD_START - local_engine::LocalExecutor::removeExecutor(executor_address); auto *executor = reinterpret_cast(executor_address); executor->cancel(); - LOG_INFO(&Poco::Logger::get("jni"), "Cancel LocalExecutor {}", reinterpret_cast(executor)); + LOG_INFO(&Poco::Logger::get("jni"), "Cancel LocalExecutor {}", reinterpret_cast(executor)); LOCAL_ENGINE_JNI_METHOD_END(env, ) } JNIEXPORT void Java_org_apache_gluten_vectorized_BatchIterator_nativeClose(JNIEnv * env, jobject /*obj*/, jlong executor_address) { LOCAL_ENGINE_JNI_METHOD_START - local_engine::LocalExecutor::removeExecutor(executor_address); auto *executor = reinterpret_cast(executor_address); LOG_INFO(&Poco::Logger::get("jni"), "Finalize LocalExecutor {}", reinterpret_cast(executor)); delete executor; @@ -1262,8 +1259,7 @@ Java_org_apache_gluten_vectorized_SimpleExpressionEval_createNativeInstance(JNIE const auto plan_a = local_engine::getByteArrayElementsSafe(env, plan); const std::string::size_type plan_size = plan_a.length(); local_engine::LocalExecutor * executor - = parser.createExecutor({reinterpret_cast(plan_a.elems()), plan_size}).release(); - local_engine::LocalExecutor::addExecutor(executor); + = parser.createExecutor({reinterpret_cast(plan_a.elems()), plan_size}).release(); return reinterpret_cast(executor); LOCAL_ENGINE_JNI_METHOD_END(env, -1) } @@ -1271,7 +1267,6 @@ Java_org_apache_gluten_vectorized_SimpleExpressionEval_createNativeInstance(JNIE JNIEXPORT void Java_org_apache_gluten_vectorized_SimpleExpressionEval_nativeClose(JNIEnv * env, jclass, jlong instance) { LOCAL_ENGINE_JNI_METHOD_START - local_engine::LocalExecutor::removeExecutor(instance); local_engine::LocalExecutor * executor = reinterpret_cast(instance); delete executor; LOCAL_ENGINE_JNI_METHOD_END(env, ) From a8e8700ef885b7d2335274aff32a16011bbb4489 Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Wed, 3 Jul 2024 20:23:29 +0800 Subject: [PATCH 392/402] [VL] Daily Update Velox Version (2024_07_03) (#6315) 4a0ce3e43 by mwish, Replace MappedMemory::Allocation with memory::Allocation in comments (10336) 49407ff9f by rui-mo, Add argument generator for Presto truncate decimal function (10245) b29176d6b by rui-mo, Fix spark-server docker image (10356) f5bfd1eed by Wei He, Fix incorrect result of approx_percentile in window operations (10368) 8fcd7c58e by zhli1142015, Improve the performance of hash Spark functions for scalar types through auto-vectorization (10301) c12aa738b by hengjiang.ly, Fix GroupIdNode`s groupIdName to const method (10345) 64dc8f33f by Pedro Eugenio Rocha Pedreira, Code formatting in velox/dwio/common (10369) c265fcf16 by Pedro Eugenio Rocha Pedreira, Cosmetic refactor on merge join supported types (10366) --- ep/build-velox/src/get_velox.sh | 2 +- .../org/apache/spark/sql/GlutenJsonFunctionsSuite.scala | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 54ba3e070e28..6cd62332a3cc 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_07_02 +VELOX_BRANCH=2024_07_03 VELOX_HOME="" #Set on run gluten on HDFS diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala index 24963f89d3fb..5a28031b6c7a 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenJsonFunctionsSuite.scala @@ -89,13 +89,13 @@ class GlutenJsonFunctionsSuite extends JsonFunctionsSuite with GlutenSQLTestsTra runTest("[\"a\",\"b\"]", "$[1]", "b") runTest("[[\"a\",\"b\"]]", "$[0][1]", "b") - runTest("[1,2,3]", "[0]", "1") - // runTest("[1,2,3]", "$0", null) crashes in velox + runTest("[1,2,3]", "[0]", null) + runTest("[1,2,3]", "$0", null) runTest("[1,2,3]", "0", null) runTest("[1,2,3]", "$.", null) - // runTest("[1,2,3]", "$", "[1,2,3]") crashes in velox - // runTest("{\"a\":4}", "$", "{\"a\":4}") crashes in velox + runTest("[1,2,3]", "$", "[1,2,3]") + runTest("{\"a\":4}", "$", "{\"a\":4}") def runTest(json: String, path: String, exp: String): Unit = { checkAnswer(Seq(json).toDF().selectExpr(s"get_json_object(value, '$path')"), Row(exp)) From 3986fd59555ea03347951de37054c3f11c0606b9 Mon Sep 17 00:00:00 2001 From: Kaifei Yi Date: Thu, 4 Jul 2024 12:51:51 +0800 Subject: [PATCH 393/402] [CELEBORN] support celeborn 0.5.0 (#6264) Co-authored-by: yikaifei --- .github/workflows/velox_docker.yml | 6 ++++-- docs/get-started/ClickHouse.md | 2 +- docs/get-started/Velox.md | 2 +- tools/gluten-it/pom.xml | 6 ++++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index d07ceb93b3d9..c4b24598ceb9 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -532,7 +532,7 @@ jobs: fail-fast: false matrix: spark: [ "spark-3.2" ] - celeborn: [ "celeborn-0.4.1", "celeborn-0.3.2-incubating" ] + celeborn: [ "celeborn-0.5.0", "celeborn-0.4.1", "celeborn-0.3.2-incubating" ] runs-on: ubuntu-20.04 container: ubuntu:22.04 steps: @@ -563,8 +563,10 @@ jobs: - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }} run: | EXTRA_PROFILE="" - if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then + if [ "${{ matrix.celeborn }}" = "celeborn-0.4.1" ]; then EXTRA_PROFILE="-Pceleborn-0.4" + elif [ "${{ matrix.celeborn }}" = "celeborn-0.5.0" ]; then + EXTRA_PROFILE="-Pceleborn-0.5" fi echo "EXTRA_PROFILE: ${EXTRA_PROFILE}" cd /opt && mkdir -p celeborn && \ diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md index 38ce048fe0de..f0b7fc13b297 100644 --- a/docs/get-started/ClickHouse.md +++ b/docs/get-started/ClickHouse.md @@ -629,7 +629,7 @@ public read-only account：gluten/hN2xX3uQ4m ### Celeborn support -Gluten with clickhouse backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`. +Gluten with clickhouse backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x`, `0.4.x` and `0.5.0`. Below introduction is used to enable this feature. diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index 5f9ae2a46b19..ff3b8f4b90f4 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -222,7 +222,7 @@ Currently there are several ways to asscess S3 in Spark. Please refer [Velox S3] ## Celeborn support -Gluten with velox backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`. +Gluten with velox backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x`, `0.4.x` and `0.5.0`. Below introduction is used to enable this feature. diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index 71db637a8403..c092a0ebb0e6 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -170,5 +170,11 @@ 0.4.1 + + celeborn-0.5 + + 0.5.0 + + From 2cd7491d4050d9167514d6057aae22124cbff88c Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Thu, 4 Jul 2024 00:23:14 -0500 Subject: [PATCH 394/402] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240704) (#6327) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240704) * Fix build due to https://github.com/ClickHouse/ClickHouse/pull/58661 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- .../local-engine/Functions/SparkFunctionRegexpExtractAll.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 1630f5760187..0fb13497d01a 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,4 +1,4 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240703 -CH_COMMIT=aa71be074ad +CH_BRANCH=rebase_ch/20240704 +CH_COMMIT=f617655ccea diff --git a/cpp-ch/local-engine/Functions/SparkFunctionRegexpExtractAll.cpp b/cpp-ch/local-engine/Functions/SparkFunctionRegexpExtractAll.cpp index ca4f1002059f..68136713f59c 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionRegexpExtractAll.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionRegexpExtractAll.cpp @@ -77,7 +77,7 @@ namespace if (arguments.size() == 3) args.emplace_back(FunctionArgumentDescriptor{"index", static_cast(&isInteger), nullptr, "Integer"}); - validateFunctionArgumentTypes(*this, arguments, args); + validateFunctionArguments(*this, arguments, args); return std::make_shared(std::make_shared()); } From ffb693ed17e17b0c2a9c2aff4719ea8309df8e89 Mon Sep 17 00:00:00 2001 From: j7nhai <146867566+j7nhai@users.noreply.github.com> Date: Thu, 4 Jul 2024 17:22:20 +0800 Subject: [PATCH 395/402] [VL] Fix build Velox script incorrectly judged as successful when run make (#6331) --- ep/build-velox/src/build_velox.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index b55f65a98e9e..47d0398c0830 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -109,6 +109,9 @@ function compile { fi fi + # Maybe there is some set option in velox setup script. Run set command again. + set -exu + CXX_FLAGS='-Wno-missing-field-initializers' COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF" if [ $BUILD_TEST_UTILS == "ON" ]; then From 7fc385dda81d4a659d36d287b49d232b9504c0b0 Mon Sep 17 00:00:00 2001 From: Zhichao Zhang Date: Thu, 4 Jul 2024 18:45:28 +0800 Subject: [PATCH 396/402] [GLUTEN-6334][CH] Support ntile window function (#6335) [CH] Support ntile window function Close #6334. --- .../backendsapi/clickhouse/CHBackend.scala | 4 +- .../clickhouse/CHSparkPlanExecApi.scala | 18 +++++++- ...enClickHouseTPCHSaltNullParquetSuite.scala | 16 +++++++ .../CommonAggregateFunctionParser.cpp | 3 -- .../aggregate_function_parser/NtileParser.cpp | 42 +++++++++++++++++++ .../aggregate_function_parser/NtileParser.h | 34 +++++++++++++++ 6 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.cpp create mode 100644 cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.h diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index cdca1b031a91..d369b8c1626f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -26,7 +26,7 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat._ import org.apache.spark.SparkEnv import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, DenseRank, Expression, Lag, Lead, Literal, NamedExpression, Rank, RowNumber} +import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning} import org.apache.spark.sql.execution.SparkPlan @@ -237,7 +237,7 @@ object CHBackendSettings extends BackendSettingsApi with Logging { } wExpression.windowFunction match { - case _: RowNumber | _: AggregateExpression | _: Rank | _: DenseRank => + case _: RowNumber | _: AggregateExpression | _: Rank | _: DenseRank | _: NTile => allSupported = allSupported case l: Lag => checkLagOrLead(l.third) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 44aeba021557..add82cbb591d 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -704,7 +704,7 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { val columnName = s"${aliasExpr.name}_${aliasExpr.exprId.id}" val wExpression = aliasExpr.child.asInstanceOf[WindowExpression] wExpression.windowFunction match { - case wf @ (RowNumber() | Rank(_) | DenseRank(_) | CumeDist() | PercentRank(_)) => + case wf @ (RowNumber() | Rank(_) | DenseRank(_)) => val aggWindowFunc = wf.asInstanceOf[AggregateWindowFunction] val frame = aggWindowFunc.frame.asInstanceOf[SpecifiedWindowFrame] val windowFunctionNode = ExpressionBuilder.makeWindowFunction( @@ -795,6 +795,22 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { originalInputAttributes.asJava ) windowExpressionNodes.add(windowFunctionNode) + case wf @ NTile(buckets: Expression) => + val frame = wExpression.windowSpec.frameSpecification.asInstanceOf[SpecifiedWindowFrame] + val childrenNodeList = new JArrayList[ExpressionNode]() + val literal = buckets.asInstanceOf[Literal] + childrenNodeList.add(LiteralTransformer(literal).doTransform(args)) + val windowFunctionNode = ExpressionBuilder.makeWindowFunction( + WindowFunctionsBuilder.create(args, wf).toInt, + childrenNodeList, + columnName, + ConverterUtils.getTypeNode(wf.dataType, wf.nullable), + frame.upper, + frame.lower, + frame.frameType.sql, + originalInputAttributes.asJava + ) + windowExpressionNodes.add(windowFunctionNode) case _ => throw new GlutenNotSupportException( "unsupported window function type: " + diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala index c0f37b08616e..b0d3e1bdb866 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -978,6 +978,22 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr compareResultsAgainstVanillaSpark(sql, true, { _ => }) } + test("window ntile") { + val sql = + """ + | select n_regionkey, n_nationkey, + | first_value(n_nationkey) over (partition by n_regionkey order by n_nationkey) as + | first_v, + | ntile(4) over (partition by n_regionkey order by n_nationkey) as ntile_v + | from + | ( + | select n_regionkey, if(n_nationkey = 1, null, n_nationkey) as n_nationkey from nation + | ) as t + | order by n_regionkey, n_nationkey + """.stripMargin + compareResultsAgainstVanillaSpark(sql, true, { _ => }) + } + test("window first value with nulls") { val sql = """ diff --git a/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp b/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp index 1619c74106d1..e7d6e1b9bd73 100644 --- a/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp +++ b/cpp-ch/local-engine/Parser/aggregate_function_parser/CommonAggregateFunctionParser.cpp @@ -42,8 +42,5 @@ REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(LastIgnoreNull, last_ignore_null, last REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(DenseRank, dense_rank, dense_rank) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(Rank, rank, rank) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(RowNumber, row_number, row_number) -REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(Ntile, ntile, ntile) -REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(PercentRank, percent_rank, percent_rank) -REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(CumeDist, cume_dist, cume_dist) REGISTER_COMMON_AGGREGATE_FUNCTION_PARSER(CountDistinct, count_distinct, uniqExact) } diff --git a/cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.cpp b/cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.cpp new file mode 100644 index 000000000000..49a59c6570fb --- /dev/null +++ b/cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.cpp @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "NtileParser.h" +#include +#include +#include + +namespace local_engine +{ +DB::ActionsDAG::NodeRawConstPtrs +NtileParser::parseFunctionArguments(const CommonFunctionInfo & func_info, const String & /*ch_func_name*/, DB::ActionsDAGPtr & actions_dag) const +{ + if (func_info.arguments.size() != 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function ntile takes exactly one argument"); + DB::ActionsDAG::NodeRawConstPtrs args; + + const auto & arg0 = func_info.arguments[0].value(); + auto [data_type, field] = parseLiteral(arg0.literal()); + if (!(DB::WhichDataType(data_type).isInt32())) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "ntile's argument must be i32"); + Int32 field_index = static_cast(field.get()); + // For CH, the data type of the args[0] must be the UInt32 + const auto * index_node = addColumnToActionsDAG(actions_dag, std::make_shared(), field_index); + args.emplace_back(index_node); + return args; +} +AggregateFunctionParserRegister ntile_register; +} diff --git a/cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.h b/cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.h new file mode 100644 index 000000000000..441de2353247 --- /dev/null +++ b/cpp-ch/local-engine/Parser/aggregate_function_parser/NtileParser.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include + +namespace local_engine +{ +class NtileParser : public AggregateFunctionParser +{ +public: + explicit NtileParser(SerializedPlanParser * plan_parser_) : AggregateFunctionParser(plan_parser_) { } + ~NtileParser() override = default; + static constexpr auto name = "ntile"; + String getName() const override { return name; } + String getCHFunctionName(const CommonFunctionInfo &) const override { return "ntile"; } + String getCHFunctionName(DB::DataTypes &) const override { return "ntile"; } + DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments( + const CommonFunctionInfo & func_info, const String & ch_func_name, DB::ActionsDAGPtr & actions_dag) const override; +}; +} From a091ac5010968274f3f8b561d7e36054f92ed151 Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Thu, 4 Jul 2024 19:18:26 +0800 Subject: [PATCH 397/402] [CORE] Drop redundant partial sort which has pre-project when offloading sort agg (#6294) --- .../VeloxAggregateFunctionsSuite.scala | 26 +++++++++++++++++++ .../apache/gluten/execution/SortUtils.scala | 24 ++++++++++++++--- .../columnar/OffloadSingleNode.scala | 8 +----- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index ae6306cc0d4a..992106d131e6 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -1135,6 +1135,32 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu df.select(max(col("txn"))).collect } + + test("drop redundant partial sort which has pre-project when offload sortAgg") { + // Spark 3.2 does not have this configuration, but it does not affect the test results. + withSQLConf("spark.sql.test.forceApplySortAggregate" -> "true") { + withTempView("t1") { + Seq((-1, 2), (-1, 3), (2, 3), (3, 4), (-3, 5), (4, 5)) + .toDF("c1", "c2") + .createOrReplaceTempView("t1") + runQueryAndCompare("select c2, sum(if(c1<0,0,c1)) from t1 group by c2") { + df => + { + assert( + getExecutedPlan(df).count( + plan => { + plan.isInstanceOf[HashAggregateExecTransformer] + }) == 2) + assert( + getExecutedPlan(df).count( + plan => { + plan.isInstanceOf[SortExecTransformer] + }) == 0) + } + } + } + } + } } class VeloxAggregateFunctionsDefaultSuite extends VeloxAggregateFunctionsSuite { diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala index 2c0ad1b0a59a..b01c71738e75 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/SortUtils.scala @@ -23,11 +23,27 @@ import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan} object SortUtils { def dropPartialSort(plan: SparkPlan): SparkPlan = plan match { case RewrittenNodeWall(p) => RewrittenNodeWall(dropPartialSort(p)) - case sort: SortExec if !sort.global => sort.child + case PartialSortLike(child) => child // from pre/post project-pulling - case ProjectExec(_, SortExec(_, false, ProjectExec(_, p), _)) - if plan.outputSet == p.outputSet => - p + case ProjectLike(PartialSortLike(ProjectLike(child))) if plan.outputSet == child.outputSet => + child + case ProjectLike(PartialSortLike(child)) => plan.withNewChildren(Seq(child)) case _ => plan } } + +object PartialSortLike { + def unapply(plan: SparkPlan): Option[SparkPlan] = plan match { + case sort: SortExecTransformer if !sort.global => Some(sort.child) + case sort: SortExec if !sort.global => Some(sort.child) + case _ => None + } +} + +object ProjectLike { + def unapply(plan: SparkPlan): Option[SparkPlan] = plan match { + case project: ProjectExecTransformer => Some(project.child) + case project: ProjectExec => Some(project.child) + case _ => None + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala index 7a4222b5cb38..62c72af792e9 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/OffloadSingleNode.scala @@ -425,13 +425,7 @@ object OffloadOthers { ColumnarCoalesceExec(plan.numPartitions, plan.child) case plan: SortAggregateExec => logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") - HashAggregateExecBaseTransformer.from(plan) { - case sort: SortExecTransformer if !sort.global => - sort.child - case sort: SortExec if !sort.global => - sort.child - case other => other - } + HashAggregateExecBaseTransformer.from(plan)(SortUtils.dropPartialSort) case plan: ObjectHashAggregateExec => logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") HashAggregateExecBaseTransformer.from(plan)() From 01526e62e84b74dd48084ea36d55df3467344c8d Mon Sep 17 00:00:00 2001 From: Mingliang Zhu Date: Thu, 4 Jul 2024 19:20:02 +0800 Subject: [PATCH 398/402] [VL] RAS: Remove NoopFilter that has same output schema with child (#6324) --- .../extension/columnar/enumerated/RemoveFilter.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala index e2b8439fd218..8b8441e8d6ce 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RemoveFilter.scala @@ -41,9 +41,12 @@ object RemoveFilter extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = { val filter = node.asInstanceOf[FilterExecTransformerBase] if (filter.isNoop()) { - val out = NoopFilter(filter.child, filter.output) - out.copyTagsFrom(filter) - return List(out) + if (filter.output != filter.child.output) { + val out = NoopFilter(filter.child, filter.output) + out.copyTagsFrom(filter) + return List(out) + } + return List(filter.child) } List.empty } From 663ae51f5d42415769354be465a09609441ffa01 Mon Sep 17 00:00:00 2001 From: Shuai li Date: Thu, 4 Jul 2024 20:28:27 +0800 Subject: [PATCH 399/402] [GLUTEN-6333][CH] Support rangepartitioning by timestamptype (#6336) [CH] Support rangepartitioning by timestamptype --- .../RangePartitionerBoundsGenerator.scala | 2 ++ ...tenClickHouseDatetimeExpressionSuite.scala | 22 +++++++++++++++++++ cpp-ch/local-engine/Parser/TypeParser.cpp | 3 ++- .../local-engine/Shuffle/SelectorBuilder.cpp | 5 +++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/utils/RangePartitionerBoundsGenerator.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/utils/RangePartitionerBoundsGenerator.scala index 61fbc86b36cc..87c6ae343d4c 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/utils/RangePartitionerBoundsGenerator.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/utils/RangePartitionerBoundsGenerator.scala @@ -199,6 +199,7 @@ class RangePartitionerBoundsGenerator[K: Ordering: ClassTag, V]( case d: DecimalType => val decimal = row.getDecimal(i, d.precision, d.scale).toString() node.put("value", decimal) + case _: TimestampType => node.put("value", row.getLong(i)) case _ => throw new IllegalArgumentException( s"Unsupported data type ${ordering.dataType.toString}") @@ -244,6 +245,7 @@ object RangePartitionerBoundsGenerator { case _: StringType => true case _: DateType => true case _: DecimalType => true + case _: TimestampType => true case _ => false } } diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala index 53416607521e..a1749efb18b2 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDatetimeExpressionSuite.scala @@ -162,4 +162,26 @@ class GlutenClickHouseDatetimeExpressionSuite |""".stripMargin compareResultsAgainstVanillaSpark(sql, true, { _ => }) } + + test("support range partition by timestamp") { + import testImplicits._ + val df = Seq( + (1, Timestamp.valueOf("2015-07-22 10:01:40.123456")), + (2, Timestamp.valueOf("2014-12-31 05:29:06.123456")), + (3, Timestamp.valueOf("2015-07-22 16:01:40.123456")), + (4, Timestamp.valueOf("2012-02-29 23:01:40.123456")) + ).toDF("i", "t") + + df.createOrReplaceTempView("test") + + val sql = + s""" + | select + | /** repartition(2) */ + | * + | from test + | order by t + |""".stripMargin + compareResultsAgainstVanillaSpark(sql, compareResult = true, { _ => }) + } } diff --git a/cpp-ch/local-engine/Parser/TypeParser.cpp b/cpp-ch/local-engine/Parser/TypeParser.cpp index 3ad19bb2bd73..0d5e54bb1757 100644 --- a/cpp-ch/local-engine/Parser/TypeParser.cpp +++ b/cpp-ch/local-engine/Parser/TypeParser.cpp @@ -59,7 +59,8 @@ std::unordered_map TypeParser::type_names_mapping {"FloatType", "Float32"}, {"DoubleType", "Float64"}, {"StringType", "String"}, - {"DateType", "Date32"}}; + {"DateType", "Date32"}, + {"TimestampType", "DateTime64"}}; String TypeParser::getCHTypeName(const String & spark_type_name) { diff --git a/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp b/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp index 7e3642dacd52..6804770c34c1 100644 --- a/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp +++ b/cpp-ch/local-engine/Shuffle/SelectorBuilder.cpp @@ -291,6 +291,11 @@ void RangeSelectorBuilder::initRangeBlock(Poco::JSON::Array::Ptr range_bounds) int val = field_value.convert(); col->insert(val); } + else if (const auto * timestamp = dynamic_cast(type_info.inner_type.get())) + { + auto value = field_value.convert(); + col->insert(DecimalField(value, 6)); + } else if (const auto * decimal32 = dynamic_cast *>(type_info.inner_type.get())) { auto value = decimal32->parseFromString(field_value.convert()); From f85b3d68c901d9f6b7016fd6121c1e8cacd6d4a6 Mon Sep 17 00:00:00 2001 From: zhixingheyi-tian Date: Thu, 4 Jul 2024 22:57:52 +0800 Subject: [PATCH 400/402] [VL] Support tencentos 2.4 (#5207) Co-authored-by: xiangxshen --- .../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 4 +++- dev/package.sh | 4 +++- ep/build-velox/src/build_velox.sh | 7 +++++++ ep/build-velox/src/get_velox.sh | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index e1abbdd7c6b7..e5c3cb084819 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -78,6 +78,8 @@ class VeloxListenerApi extends ListenerApi { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Anolis") && systemVersion.startsWith("7")) { new SharedLibraryLoaderCentos7 + } else if (system.contains("tencentos") && system.contains("2.4")) { + new SharedLibraryLoaderCentos7 } else if (system.contains("tencentos") && system.contains("3.2")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("9")) { @@ -94,7 +96,7 @@ class VeloxListenerApi extends ListenerApi { throw new GlutenException( s"Found unsupported OS($systemName, $systemVersion)! Currently, Gluten's Velox backend" + " only supports Ubuntu 20.04/22.04, CentOS 7/8, " + - "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 3.2, RedHat 7/8/9, " + + "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 2.4/3.2, RedHat 7/8, " + "Debian 11/12.") } } diff --git a/dev/package.sh b/dev/package.sh index 1b9ca85e9590..7e7e793bdabd 100755 --- a/dev/package.sh +++ b/dev/package.sh @@ -68,7 +68,9 @@ elif [ "$LINUX_OS" == "alinux" ]; then process_setup_centos_7 fi elif [ "$LINUX_OS" == "tencentos" ]; then - if [ "$VERSION" == "3.2" ]; then + if [ "$VERSION" == "2.4" ]; then + process_setup_centos_7 + elif [ "$VERSION" == "3.2" ]; then process_setup_centos_8 fi elif [ "$LINUX_OS" == "debian" ]; then diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 47d0398c0830..747eec3f8b6f 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -265,6 +265,13 @@ function setup_linux { esac elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then case "$LINUX_VERSION_ID" in + 2.4) + scripts/setup-centos7.sh + set +u + export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH + source /opt/rh/devtoolset-9/enable + set -u + ;; 3.2) scripts/setup-centos8.sh ;; *) echo "Unsupported tencentos version: $LINUX_VERSION_ID" diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 6cd62332a3cc..9193d7f89848 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -286,6 +286,7 @@ function setup_linux { esac elif [[ "$LINUX_DISTRIBUTION" == "tencentos" ]]; then case "$LINUX_VERSION_ID" in + 2.4) process_setup_centos7 ;; 3.2) process_setup_tencentos32 ;; *) echo "Unsupport tencentos version: $LINUX_VERSION_ID" From ff0b4733a8282476bbb4dbcca2d46458ca04b36b Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Fri, 5 Jul 2024 00:39:12 +0800 Subject: [PATCH 401/402] [VL] Daily Update Velox Version (2024_07_04) (#6328) 26f001441 by xiaoxmeng, Fix tsan race in cache fuzzer test (10391) 1705bde41 by xiaoxmeng, Add exchange request count metrics (10389) 02cbe7476 by Bikramjeet Vig, Fix NaN handling in comparison functions (10165) decd91eb5 by Kevin Wilfong, Call commitNull in SimpleFunctionAdapter on exceptions (10377) a03396845 by Kevin Wilfong, StringWriter needs to implement finalizeNull (10376) 0cb715ea3 by youxiduo, Reject duplicate sorting keys (10040) 4e39b06c7 by gaoyangxiaozhu, Add Spark raise_error function (10110) 63ccecaa2 by rui-mo, Check determinism using function name only (10241) --- .../gluten/execution/TestOperator.scala | 20 +++++++++---------- ep/build-velox/src/get_velox.sh | 2 +- .../GlutenSQLWindowFunctionSuite.scala | 2 +- .../GlutenSQLWindowFunctionSuite.scala | 2 +- .../utils/velox/VeloxTestSettings.scala | 1 + .../GlutenSQLWindowFunctionSuite.scala | 2 +- .../utils/velox/VeloxTestSettings.scala | 1 + .../GlutenSQLWindowFunctionSuite.scala | 6 +++--- 8 files changed, 19 insertions(+), 17 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index c010b9128ce1..c6e72197d6c7 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -422,18 +422,18 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } // Test same partition/ordering keys. - runQueryAndCompare( - "select avg(l_partkey) over" + - " (partition by l_suppkey order by l_suppkey) from lineitem ") { - checkGlutenOperatorMatch[WindowExecTransformer] - } +// runQueryAndCompare( +// "select avg(l_partkey) over" + +// " (partition by l_suppkey order by l_suppkey) from lineitem ") { +// checkGlutenOperatorMatch[WindowExecTransformer] +// } // Test overlapping partition/ordering keys. - runQueryAndCompare( - "select avg(l_partkey) over" + - " (partition by l_suppkey order by l_suppkey, l_orderkey) from lineitem ") { - checkGlutenOperatorMatch[WindowExecTransformer] - } +// runQueryAndCompare( +// "select avg(l_partkey) over" + +// " (partition by l_suppkey order by l_suppkey, l_orderkey) from lineitem ") { +// checkGlutenOperatorMatch[WindowExecTransformer] +// } } } } diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 9193d7f89848..07694a36102d 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_07_03 +VELOX_BRANCH=2024_07_04 VELOX_HOME="" #Set on run gluten on HDFS diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala index 6665174207b0..e26c1dc41a9a 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala @@ -47,7 +47,7 @@ class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQL Row(95337, 12, decimal(915.61)) ) - testGluten("Literal in window partition by and sort") { + ignoreGluten("Literal in window partition by and sort") { withTable("customer") { val rdd = spark.sparkContext.parallelize(customerData) val customerDF = spark.createDataFrame(rdd, customerSchema) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala index 6665174207b0..e26c1dc41a9a 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala @@ -47,7 +47,7 @@ class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQL Row(95337, 12, decimal(915.61)) ) - testGluten("Literal in window partition by and sort") { + ignoreGluten("Literal in window partition by and sort") { withTable("customer") { val rdd = spark.sparkContext.parallelize(customerData) val customerDF = spark.createDataFrame(rdd, customerSchema) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 0da19922ffda..941f7b89b2d5 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1182,6 +1182,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataFrameToSchemaSuite] enableSuite[GlutenDatasetUnpivotSuite] enableSuite[GlutenLateralColumnAliasSuite] + .exclude("Aggregate expressions containing no aggregate or grouping expressions still resolves") enableSuite[GlutenParametersSuite] enableSuite[GlutenResolveDefaultColumnsSuite] enableSuite[GlutenSubqueryHintPropagationSuite] diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala index 6665174207b0..e26c1dc41a9a 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala @@ -47,7 +47,7 @@ class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQL Row(95337, 12, decimal(915.61)) ) - testGluten("Literal in window partition by and sort") { + ignoreGluten("Literal in window partition by and sort") { withTable("customer") { val rdd = spark.sparkContext.parallelize(customerData) val customerDF = spark.createDataFrame(rdd, customerSchema) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index e54aca34ec75..f3bd7ff6c752 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1197,6 +1197,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDataFrameToSchemaSuite] enableSuite[GlutenDatasetUnpivotSuite] enableSuite[GlutenLateralColumnAliasSuite] + .exclude("Aggregate expressions containing no aggregate or grouping expressions still resolves") enableSuite[GlutenParametersSuite] enableSuite[GlutenResolveDefaultColumnsSuite] enableSuite[GlutenSubqueryHintPropagationSuite] diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala index 89a4351744ef..e61c084ab7db 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenSQLWindowFunctionSuite.scala @@ -47,7 +47,7 @@ class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQL Row(95337, 12, decimal(915.61)) ) - testGluten("Literal in window partition by and sort") { + ignoreGluten("Literal in window partition by and sort") { withTable("customer") { val rdd = spark.sparkContext.parallelize(customerData) val customerDF = spark.createDataFrame(rdd, customerSchema) @@ -93,7 +93,7 @@ class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQL } } - testGluten("Filter on row number") { + ignoreGluten("Filter on row number") { withTable("customer") { val rdd = spark.sparkContext.parallelize(customerData) val customerDF = spark.createDataFrame(rdd, customerSchema) @@ -137,7 +137,7 @@ class GlutenSQLWindowFunctionSuite extends SQLWindowFunctionSuite with GlutenSQL } } - testGluten("Filter on rank") { + ignoreGluten("Filter on rank") { withTable("customer") { val rdd = spark.sparkContext.parallelize(customerData) val customerDF = spark.createDataFrame(rdd, customerSchema) From 995145e93cb8c930501e69156ca57211d9932d2a Mon Sep 17 00:00:00 2001 From: lgbo Date: Fri, 5 Jul 2024 08:34:20 +0800 Subject: [PATCH 402/402] support sort_array (#6323) --- .../clickhouse/CHSparkPlanExecApi.scala | 9 + .../Functions/SparkFunctionArraySort.cpp | 223 ++++++++++++++---- .../Functions/SparkFunctionSortArray.cpp | 88 +++++++ ...onArraySort.h => SparkFunctionSortArray.h} | 14 +- .../arrayHighOrderFunctions.cpp | 144 +++++++++++ .../scalar_function_parser/sortArray.cpp | 4 +- .../gluten/backendsapi/SparkPlanExecApi.scala | 9 + .../expression/ExpressionConverter.scala | 13 + .../expression/ExpressionMappings.scala | 1 + .../gluten/expression/ExpressionNames.scala | 1 + 10 files changed, 454 insertions(+), 52 deletions(-) create mode 100644 cpp-ch/local-engine/Functions/SparkFunctionSortArray.cpp rename cpp-ch/local-engine/Functions/{SparkFunctionArraySort.h => SparkFunctionSortArray.h} (86%) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index add82cbb591d..f5feade886b9 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -876,6 +876,15 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(argument, function), expr) } + /** Transform array sort to Substrait. */ + override def genArraySortTransformer( + substraitExprName: String, + argument: ExpressionTransformer, + function: ExpressionTransformer, + expr: ArraySort): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(argument, function), expr) + } + override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = generate override def genPostProjectForGenerate(generate: GenerateExec): SparkPlan = generate diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArraySort.cpp b/cpp-ch/local-engine/Functions/SparkFunctionArraySort.cpp index 126b84eaaf95..1371ec60e179 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionArraySort.cpp +++ b/cpp-ch/local-engine/Functions/SparkFunctionArraySort.cpp @@ -14,75 +14,212 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -namespace DB +namespace DB::ErrorCodes { + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int TYPE_MISMATCH; + extern const int ILLEGAL_COLUMN; +} -namespace ErrorCodes +/// The usage of `arraySort` in CH is different from Spark's `sort_array` function. +/// We need to implement a custom function to sort arrays. +namespace local_engine { - extern const int LOGICAL_ERROR; -} -namespace +struct LambdaLess { + const DB::IColumn & column; + DB::DataTypePtr type; + const DB::ColumnFunction & lambda; + explicit LambdaLess(const DB::IColumn & column_, DB::DataTypePtr type_, const DB::ColumnFunction & lambda_) + : column(column_), type(type_), lambda(lambda_) {} + + /// May not efficient + bool operator()(size_t lhs, size_t rhs) const + { + /// The column name seems not matter. + auto left_value_col = DB::ColumnWithTypeAndName(oneRowColumn(lhs), type, "left"); + auto right_value_col = DB::ColumnWithTypeAndName(oneRowColumn(rhs), type, "right"); + auto cloned_lambda = lambda.cloneResized(1); + auto * lambda_ = typeid_cast(cloned_lambda.get()); + lambda_->appendArguments({std::move(left_value_col), std::move(right_value_col)}); + auto compare_res_col = lambda_->reduce(); + DB::Field field; + compare_res_col.column->get(0, field); + return field.get() < 0; + } +private: + ALWAYS_INLINE DB::ColumnPtr oneRowColumn(size_t i) const + { + auto res = column.cloneEmpty(); + res->insertFrom(column, i); + return std::move(res); + } +}; -template struct Less { - const IColumn & column; + const DB::IColumn & column; - explicit Less(const IColumn & column_) : column(column_) { } + explicit Less(const DB::IColumn & column_) : column(column_) { } bool operator()(size_t lhs, size_t rhs) const { - if constexpr (positive) - /* - Note: We use nan_direction_hint=-1 for ascending sort to make NULL the least value. - However, NaN is also considered the least value, - which results in different sorting results compared to Spark since Spark treats NaN as the greatest value. - For now, we are temporarily ignoring this issue because cases with NaN are rare, - and aligning with Spark would require tricky modifications to the CH underlying code. - */ - return column.compareAt(lhs, rhs, column, -1) < 0; - else - return column.compareAt(lhs, rhs, column, -1) > 0; + return column.compareAt(lhs, rhs, column, 1) < 0; } }; -} - -template -ColumnPtr SparkArraySortImpl::execute( - const ColumnArray & array, - ColumnPtr mapped, - const ColumnWithTypeAndName * fixed_arguments [[maybe_unused]]) +class FunctionSparkArraySort : public DB::IFunction { - const ColumnArray::Offsets & offsets = array.getOffsets(); +public: + static constexpr auto name = "arraySortSpark"; + static DB::FunctionPtr create(DB::ContextPtr /*context*/) { return std::make_shared(); } - size_t size = offsets.size(); - size_t nested_size = array.getData().size(); - IColumn::Permutation permutation(nested_size); + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool isSuitableForShortCircuitArgumentsExecution(const DB::DataTypesWithConstInfo &) const override { return true; } - for (size_t i = 0; i < nested_size; ++i) - permutation[i] = i; + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } - ColumnArray::Offset current_offset = 0; - for (size_t i = 0; i < size; ++i) + void getLambdaArgumentTypes(DB::DataTypes & arguments) const override { - auto next_offset = offsets[i]; - ::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); - current_offset = next_offset; + if (arguments.size() < 2) + throw DB::Exception(DB::ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function {} requires as arguments a lambda function and an array", getName()); + + if (arguments.size() > 1) + { + const auto * lambda_function_type = DB::checkAndGetDataType(arguments[0].get()); + if (!lambda_function_type || lambda_function_type->getArgumentTypes().size() != 2) + throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument of function {} must be a lambda function with 2 arguments, found {} instead.", + getName(), arguments[0]->getName()); + auto array_nesteed_type = DB::checkAndGetDataType(arguments.back().get())->getNestedType(); + DB::DataTypes lambda_args = {array_nesteed_type, array_nesteed_type}; + arguments[0] = std::make_shared(lambda_args); + } } - return ColumnArray::create(array.getData().permute(permutation, 0), array.getOffsetsPtr()); -} + DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() > 1) + { + const auto * lambda_function_type = checkAndGetDataType(arguments[0].type.get()); + if (!lambda_function_type) + throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function", getName()); + } + + return arguments.back().type; + } + + DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr &, size_t input_rows_count) const override + { + auto array_col = arguments.back().column; + auto array_type = arguments.back().type; + DB::ColumnPtr null_map = nullptr; + if (const auto * null_col = typeid_cast(array_col.get())) + { + null_map = null_col->getNullMapColumnPtr(); + array_col = null_col->getNestedColumnPtr(); + array_type = typeid_cast(array_type.get())->getNestedType(); + } + + const auto * array_col_concrete = DB::checkAndGetColumn(array_col.get()); + if (!array_col_concrete) + { + const auto * aray_col_concrete_const = DB::checkAndGetColumnConst(array_col.get()); + if (!aray_col_concrete_const) + { + throw DB::Exception(DB::ErrorCodes::ILLEGAL_COLUMN, "Expected array column, found {}", array_col->getName()); + } + array_col = DB::recursiveRemoveLowCardinality(aray_col_concrete_const->convertToFullColumn()); + array_col_concrete = DB::checkAndGetColumn(array_col.get()); + } + auto array_nested_type = DB::checkAndGetDataType(array_type.get())->getNestedType(); + + DB::ColumnPtr sorted_array_col = nullptr; + if (arguments.size() > 1) + sorted_array_col = executeWithLambda(*array_col_concrete, array_nested_type, *checkAndGetColumn(arguments[0].column.get())); + else + sorted_array_col = executeWithoutLambda(*array_col_concrete); + + if (null_map) + { + sorted_array_col = DB::ColumnNullable::create(sorted_array_col, null_map); + } + return sorted_array_col; + } +private: + static DB::ColumnPtr executeWithLambda(const DB::ColumnArray & array_col, DB::DataTypePtr array_nested_type, const DB::ColumnFunction & lambda) + { + const auto & offsets = array_col.getOffsets(); + auto rows = array_col.size(); + + size_t nested_size = array_col.getData().size(); + DB::IColumn::Permutation permutation(nested_size); + for (size_t i = 0; i < nested_size; ++i) + permutation[i] = i; + + DB::ColumnArray::Offset current_offset = 0; + for (size_t i = 0; i < rows; ++i) + { + auto next_offset = offsets[i]; + ::sort(&permutation[current_offset], + &permutation[next_offset], + LambdaLess(array_col.getData(), + array_nested_type, + lambda)); + current_offset = next_offset; + } + auto res = DB::ColumnArray::create(array_col.getData().permute(permutation, 0), array_col.getOffsetsPtr()); + return res; + } + + static DB::ColumnPtr executeWithoutLambda(const DB::ColumnArray & array_col) + { + const auto & offsets = array_col.getOffsets(); + auto rows = array_col.size(); + + size_t nested_size = array_col.getData().size(); + DB::IColumn::Permutation permutation(nested_size); + for (size_t i = 0; i < nested_size; ++i) + permutation[i] = i; + + DB::ColumnArray::Offset current_offset = 0; + for (size_t i = 0; i < rows; ++i) + { + auto next_offset = offsets[i]; + ::sort(&permutation[current_offset], + &permutation[next_offset], + Less(array_col.getData())); + current_offset = next_offset; + } + auto res = DB::ColumnArray::create(array_col.getData().permute(permutation, 0), array_col.getOffsetsPtr()); + return res; + } + + String getName() const override + { + return name; + } + +}; REGISTER_FUNCTION(ArraySortSpark) { - factory.registerFunction(); - factory.registerFunction(); + factory.registerFunction(); } - } diff --git a/cpp-ch/local-engine/Functions/SparkFunctionSortArray.cpp b/cpp-ch/local-engine/Functions/SparkFunctionSortArray.cpp new file mode 100644 index 000000000000..42b88fbce730 --- /dev/null +++ b/cpp-ch/local-engine/Functions/SparkFunctionSortArray.cpp @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace +{ + +template +struct Less +{ + const IColumn & column; + + explicit Less(const IColumn & column_) : column(column_) { } + + bool operator()(size_t lhs, size_t rhs) const + { + if constexpr (positive) + /* + Note: We use nan_direction_hint=-1 for ascending sort to make NULL the least value. + However, NaN is also considered the least value, + which results in different sorting results compared to Spark since Spark treats NaN as the greatest value. + For now, we are temporarily ignoring this issue because cases with NaN are rare, + and aligning with Spark would require tricky modifications to the CH underlying code. + */ + return column.compareAt(lhs, rhs, column, -1) < 0; + else + return column.compareAt(lhs, rhs, column, -1) > 0; + } +}; + +} + +template +ColumnPtr SparkSortArrayImpl::execute( + const ColumnArray & array, + ColumnPtr mapped, + const ColumnWithTypeAndName * fixed_arguments [[maybe_unused]]) +{ + const ColumnArray::Offsets & offsets = array.getOffsets(); + + size_t size = offsets.size(); + size_t nested_size = array.getData().size(); + IColumn::Permutation permutation(nested_size); + + for (size_t i = 0; i < nested_size; ++i) + permutation[i] = i; + + ColumnArray::Offset current_offset = 0; + for (size_t i = 0; i < size; ++i) + { + auto next_offset = offsets[i]; + ::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); + current_offset = next_offset; + } + + return ColumnArray::create(array.getData().permute(permutation, 0), array.getOffsetsPtr()); +} + +REGISTER_FUNCTION(SortArraySpark) +{ + factory.registerFunction(); + factory.registerFunction(); +} + +} diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArraySort.h b/cpp-ch/local-engine/Functions/SparkFunctionSortArray.h similarity index 86% rename from cpp-ch/local-engine/Functions/SparkFunctionArraySort.h rename to cpp-ch/local-engine/Functions/SparkFunctionSortArray.h index 9ce48f9c0baf..18c2128c0258 100644 --- a/cpp-ch/local-engine/Functions/SparkFunctionArraySort.h +++ b/cpp-ch/local-engine/Functions/SparkFunctionSortArray.h @@ -32,7 +32,7 @@ namespace ErrorCodes /** Sort arrays, by values of its elements, or by values of corresponding elements of calculated expression (known as "schwartzsort"). */ template -struct SparkArraySortImpl +struct SparkSortArrayImpl { static bool needBoolean() { return false; } static bool needExpression() { return false; } @@ -67,16 +67,16 @@ struct SparkArraySortImpl const ColumnWithTypeAndName * fixed_arguments [[maybe_unused]] = nullptr); }; -struct NameArraySort +struct NameSortArray { - static constexpr auto name = "arraySortSpark"; + static constexpr auto name = "sortArraySpark"; }; -struct NameArrayReverseSort +struct NameReverseSortArray { - static constexpr auto name = "arrayReverseSortSpark"; + static constexpr auto name = "reverseSortArraySpark"; }; -using SparkFunctionArraySort = FunctionArrayMapped, NameArraySort>; -using SparkFunctionArrayReverseSort = FunctionArrayMapped, NameArrayReverseSort>; +using SparkFunctionSortArray = FunctionArrayMapped, NameSortArray>; +using SparkFunctionReverseSortArray = FunctionArrayMapped, NameReverseSortArray>; } diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp index 584bc0ef1e04..3811880aea63 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp @@ -151,4 +151,148 @@ class ArrayAggregate : public FunctionParser }; static FunctionParserRegister register_array_aggregate; +class ArraySort : public FunctionParser +{ +public: + static constexpr auto name = "array_sort"; + explicit ArraySort(SerializedPlanParser * plan_parser_) : FunctionParser(plan_parser_) {} + ~ArraySort() override = default; + String getName() const override { return name; } + String getCHFunctionName(const substrait::Expression_ScalarFunction & scalar_function) const override + { + return "arraySortSpark"; + } + const DB::ActionsDAG::Node * parse(const substrait::Expression_ScalarFunction & substrait_func, + DB::ActionsDAGPtr & actions_dag) const + { + auto ch_func_name = getCHFunctionName(substrait_func); + auto parsed_args = parseFunctionArguments(substrait_func, ch_func_name, actions_dag); + + if (parsed_args.size() != 2) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "array_sort function must have two arguments"); + if (isDefaultCompare(substrait_func.arguments()[1].value().scalar_function())) + { + return toFunctionNode(actions_dag, ch_func_name, {parsed_args[0]}); + } + + return toFunctionNode(actions_dag, ch_func_name, {parsed_args[1], parsed_args[0]}); + } +private: + + /// The default lambda compare function for array_sort, `array_sort(x)`. + bool isDefaultCompare(const substrait::Expression_ScalarFunction & scalar_function) const + { + String left_variable_name, right_variable_name; + auto names_types = collectLambdaArguments(*plan_parser, scalar_function); + { + auto it = names_types.begin(); + left_variable_name = it->name; + it++; + right_variable_name = it->name; + } + + auto is_function = [&](const substrait::Expression & expr, const String & function_name) { + return expr.has_scalar_function() + && *(plan_parser->getFunctionSignatureName(expr.scalar_function().function_reference())) == function_name; + }; + + auto is_variable = [&](const substrait::Expression & expr, const String & var) { + if (!is_function(expr, "namedlambdavariable")) + { + return false; + } + const auto var_expr = expr.scalar_function().arguments()[0].value(); + if (!var_expr.has_literal()) + return false; + auto [_, name] = plan_parser->parseLiteral(var_expr.literal()); + return var == name.get(); + }; + + auto is_int_value = [&](const substrait::Expression & expr, Int32 val) { + if (!expr.has_literal()) + return false; + auto [_, x] = plan_parser->parseLiteral(expr.literal()); + return val == x.get(); + }; + + auto is_variable_null = [&](const substrait::Expression & expr, const String & var) { + return is_function(expr, "is_null") && is_variable(expr.scalar_function().arguments(0).value(), var); + }; + + auto is_both_null = [&](const substrait::Expression & expr) { + return is_function(expr, "and") + && is_variable_null(expr.scalar_function().arguments(0).value(), left_variable_name) + && is_variable_null(expr.scalar_function().arguments(1).value(), right_variable_name); + }; + + auto is_left_greater_right = [&](const substrait::Expression & expr) { + if (!expr.has_if_then()) + return false; + + const auto & if_ = expr.if_then().ifs(0); + if (!is_function(if_.if_(), "gt")) + return false; + + const auto & less_args = if_.if_().scalar_function().arguments(); + return is_variable(less_args[0].value(), left_variable_name) + && is_variable(less_args[1].value(), right_variable_name) + && is_int_value(if_.then(), 1) + && is_int_value(expr.if_then().else_(), 0); + }; + + auto is_left_less_right = [&](const substrait::Expression & expr) { + if (!expr.has_if_then()) + return false; + + const auto & if_ = expr.if_then().ifs(0); + if (!is_function(if_.if_(), "lt")) + return false; + + const auto & less_args = if_.if_().scalar_function().arguments(); + return is_variable(less_args[0].value(), left_variable_name) + && is_variable(less_args[1].value(), right_variable_name) + && is_int_value(if_.then(), -1) + && is_left_greater_right(expr.if_then().else_()); + }; + + auto is_right_null_else = [&](const substrait::Expression & expr) { + if (!expr.has_if_then()) + return false; + + /// if right arg is null, return 1 + const auto & if_then = expr.if_then(); + return is_variable_null(if_then.ifs(0).if_(), right_variable_name) + && is_int_value(if_then.ifs(0).then(), -1) + && is_left_less_right(if_then.else_()); + + }; + + auto is_left_null_else = [&](const substrait::Expression & expr) { + if (!expr.has_if_then()) + return false; + + /// if left arg is null, return 1 + const auto & if_then = expr.if_then(); + return is_variable_null(if_then.ifs(0).if_(), left_variable_name) + && is_int_value(if_then.ifs(0).then(), 1) + && is_right_null_else(if_then.else_()); + }; + + auto is_if_both_null_else = [&](const substrait::Expression & expr) { + if (!expr.has_if_then()) + { + return false; + } + const auto & if_ = expr.if_then().ifs(0); + return is_both_null(if_.if_()) + && is_int_value(if_.then(), 0) + && is_left_null_else(expr.if_then().else_()); + }; + + const auto & lambda_body = scalar_function.arguments()[0].value(); + return is_if_both_null_else(lambda_body); + } +}; +static FunctionParserRegister register_array_sort; + } diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/sortArray.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/sortArray.cpp index 85416bd71864..4fd2fd4f6800 100644 --- a/cpp-ch/local-engine/Parser/scalar_function_parser/sortArray.cpp +++ b/cpp-ch/local-engine/Parser/scalar_function_parser/sortArray.cpp @@ -52,8 +52,8 @@ class FunctionParserSortArray : public FunctionParser const auto * array_arg = parsed_args[0]; const auto * order_arg = parsed_args[1]; - const auto * sort_node = toFunctionNode(actions_dag, "arraySortSpark", {array_arg}); - const auto * reverse_sort_node = toFunctionNode(actions_dag, "arrayReverseSortSpark", {array_arg}); + const auto * sort_node = toFunctionNode(actions_dag, "sortArraySpark", {array_arg}); + const auto * reverse_sort_node = toFunctionNode(actions_dag, "reverseSortArraySpark", {array_arg}); const auto * result_node = toFunctionNode(actions_dag, "if", {order_arg, sort_node, reverse_sort_node}); return result_node; diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index ff7449e2d340..a69d41d00c12 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -258,6 +258,15 @@ trait SparkPlanExecApi { throw new GlutenNotSupportException("all_match is not supported") } + /** Transform array array_sort to Substrait. */ + def genArraySortTransformer( + substraitExprName: String, + argument: ExpressionTransformer, + function: ExpressionTransformer, + expr: ArraySort): ExpressionTransformer = { + throw new GlutenNotSupportException("array_sort(on array) is not supported") + } + /** Transform array exists to Substrait */ def genArrayExistsTransformer( substraitExprName: String, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index b5bcb6876e4d..805ff94900fe 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -556,6 +556,19 @@ object ExpressionConverter extends SQLConfHelper with Logging { expressionsMap), arrayTransform ) + case arraySort: ArraySort => + BackendsApiManager.getSparkPlanExecApiInstance.genArraySortTransformer( + substraitExprName, + replaceWithExpressionTransformerInternal( + arraySort.argument, + attributeSeq, + expressionsMap), + replaceWithExpressionTransformerInternal( + arraySort.function, + attributeSeq, + expressionsMap), + arraySort + ) case tryEval @ TryEval(a: Add) => BackendsApiManager.getSparkPlanExecApiInstance.genTryArithmeticTransformer( substraitExprName, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index e7e9c7ffe900..51e78a97e997 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -248,6 +248,7 @@ object ExpressionMappings { Sig[ArrayFilter](FILTER), Sig[ArrayForAll](FORALL), Sig[ArrayExists](EXISTS), + Sig[ArraySort](ARRAY_SORT), Sig[Shuffle](SHUFFLE), Sig[ZipWith](ZIP_WITH), Sig[Flatten](FLATTEN), diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 278f11922645..e3dc3a8ab0a9 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -261,6 +261,7 @@ object ExpressionNames { final val ARRAY_EXCEPT = "array_except" final val ARRAY_REPEAT = "array_repeat" final val ARRAY_REMOVE = "array_remove" + final val ARRAY_SORT = "array_sort" final val ARRAYS_ZIP = "arrays_zip" final val FILTER = "filter" final val FORALL = "forall"